{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "multihead-siamese-nets.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/tlatkowski/multihead-siamese-nets/blob/master/colab/multihead_siamese_nets.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VgtTAICkayQg",
        "colab_type": "text"
      },
      "source": [
        "**Download multihead-siamese-net code from repository:**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_bXt9MOlYcyu",
        "colab_type": "code",
        "outputId": "b7e5a878-4786-4299-a7ae-964e4f520159",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        }
      },
      "source": [
        "!git clone https://github.com/tlatkowski/multihead-siamese-nets.git"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'multihead-siamese-nets'...\n",
            "remote: Enumerating objects: 133, done.\u001b[K\n",
            "remote: Counting objects:   0% (1/133)\u001b[K\rremote: Counting objects:   1% (2/133)\u001b[K\rremote: Counting objects:   2% (3/133)\u001b[K\rremote: Counting objects:   3% (4/133)\u001b[K\rremote: Counting objects:   4% (6/133)\u001b[K\rremote: Counting objects:   5% (7/133)\u001b[K\rremote: Counting objects:   6% (8/133)\u001b[K\rremote: Counting objects:   7% (10/133)\u001b[K\rremote: Counting objects:   8% (11/133)\u001b[K\rremote: Counting objects:   9% (12/133)\u001b[K\rremote: Counting objects:  10% (14/133)\u001b[K\rremote: Counting objects:  11% (15/133)\u001b[K\rremote: Counting objects:  12% (16/133)\u001b[K\rremote: Counting objects:  13% (18/133)\u001b[K\rremote: Counting objects:  14% (19/133)\u001b[K\rremote: Counting objects:  15% (20/133)\u001b[K\rremote: Counting objects:  16% (22/133)\u001b[K\rremote: Counting objects:  17% (23/133)\u001b[K\rremote: Counting objects:  18% (24/133)\u001b[K\rremote: Counting objects:  19% (26/133)\u001b[K\rremote: Counting objects:  20% (27/133)\u001b[K\rremote: Counting objects:  21% (28/133)\u001b[K\rremote: Counting objects:  22% (30/133)\u001b[K\rremote: Counting objects:  23% (31/133)\u001b[K\rremote: Counting objects:  24% (32/133)\u001b[K\rremote: Counting objects:  25% (34/133)\u001b[K\rremote: Counting objects:  26% (35/133)\u001b[K\rremote: Counting objects:  27% (36/133)\u001b[K\rremote: Counting objects:  28% (38/133)\u001b[K\rremote: Counting objects:  29% (39/133)\u001b[K\rremote: Counting objects:  30% (40/133)\u001b[K\rremote: Counting objects:  31% (42/133)\u001b[K\rremote: Counting objects:  32% (43/133)\u001b[K\rremote: Counting objects:  33% (44/133)\u001b[K\rremote: Counting objects:  34% (46/133)\u001b[K\rremote: Counting objects:  35% (47/133)\u001b[K\rremote: Counting objects:  36% (48/133)\u001b[K\rremote: Counting objects:  37% (50/133)\u001b[K\rremote: Counting objects:  38% (51/133)\u001b[K\rremote: Counting objects:  39% (52/133)\u001b[K\rremote: Counting objects:  40% (54/133)\u001b[K\rremote: Counting objects:  41% (55/133)\u001b[K\rremote: Counting objects:  42% (56/133)\u001b[K\rremote: Counting objects:  43% (58/133)\u001b[K\rremote: Counting objects:  44% (59/133)\u001b[K\rremote: Counting objects:  45% (60/133)\u001b[K\rremote: Counting objects:  46% (62/133)\u001b[K\rremote: Counting objects:  47% (63/133)\u001b[K\rremote: Counting objects:  48% (64/133)\u001b[K\rremote: Counting objects:  49% (66/133)\u001b[K\rremote: Counting objects:  50% (67/133)\u001b[K\rremote: Counting objects:  51% (68/133)\u001b[K\rremote: Counting objects:  52% (70/133)\u001b[K\rremote: Counting objects:  53% (71/133)\u001b[K\rremote: Counting objects:  54% (72/133)\u001b[K\rremote: Counting objects:  55% (74/133)\u001b[K\rremote: Counting objects:  56% (75/133)\u001b[K\rremote: Counting objects:  57% (76/133)\u001b[K\rremote: Counting objects:  58% (78/133)\u001b[K\rremote: Counting objects:  59% (79/133)\u001b[K\rremote: Counting objects:  60% (80/133)\u001b[K\rremote: Counting objects:  61% (82/133)\u001b[K\rremote: Counting objects:  62% (83/133)\u001b[K\rremote: Counting objects:  63% (84/133)\u001b[K\rremote: Counting objects:  64% (86/133)\u001b[K\rremote: Counting objects:  65% (87/133)\u001b[K\rremote: Counting objects:  66% (88/133)\u001b[K\rremote: Counting objects:  67% (90/133)\u001b[K\rremote: Counting objects:  68% (91/133)\u001b[K\rremote: Counting objects:  69% (92/133)\u001b[K\rremote: Counting objects:  70% (94/133)\u001b[K\rremote: Counting objects:  71% (95/133)\u001b[K\rremote: Counting objects:  72% (96/133)\u001b[K\rremote: Counting objects:  73% (98/133)\u001b[K\rremote: Counting objects:  74% (99/133)\u001b[K\rremote: Counting objects:  75% (100/133)\u001b[K\rremote: Counting objects:  76% (102/133)\u001b[K\rremote: Counting objects:  77% (103/133)\u001b[K\rremote: Counting objects:  78% (104/133)\u001b[K\rremote: Counting objects:  79% (106/133)\u001b[K\rremote: Counting objects:  80% (107/133)\u001b[K\rremote: Counting objects:  81% (108/133)\u001b[K\rremote: Counting objects:  82% (110/133)\u001b[K\rremote: Counting objects:  83% (111/133)\u001b[K\rremote: Counting objects:  84% (112/133)\u001b[K\rremote: Counting objects:  85% (114/133)\u001b[K\rremote: Counting objects:  86% (115/133)\u001b[K\rremote: Counting objects:  87% (116/133)\u001b[K\rremote: Counting objects:  88% (118/133)\u001b[K\rremote: Counting objects:  89% (119/133)\u001b[K\rremote: Counting objects:  90% (120/133)\u001b[K\rremote: Counting objects:  91% (122/133)\u001b[K\rremote: Counting objects:  92% (123/133)\u001b[K\rremote: Counting objects:  93% (124/133)\u001b[K\rremote: Counting objects:  94% (126/133)\u001b[K\rremote: Counting objects:  95% (127/133)\u001b[K\rremote: Counting objects:  96% (128/133)\u001b[K\rremote: Counting objects:  97% (130/133)\u001b[K\rremote: Counting objects:  98% (131/133)\u001b[K\rremote: Counting objects:  99% (132/133)\u001b[K\rremote: Counting objects: 100% (133/133)\u001b[K\rremote: Counting objects: 100% (133/133), done.\u001b[K\n",
            "remote: Compressing objects: 100% (98/98), done.\u001b[K\n",
            "remote: Total 807 (delta 69), reused 75 (delta 34), pack-reused 674\u001b[K\n",
            "Receiving objects: 100% (807/807), 1.30 MiB | 4.38 MiB/s, done.\n",
            "Resolving deltas: 100% (462/462), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yKTs265pYoyX",
        "colab_type": "code",
        "outputId": "facfbdbf-16f1-4b01-aa37-650d98c43b69",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "!ls"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "multihead-siamese-nets\tsample_data\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FCZ7idGfbFQh",
        "colab_type": "text"
      },
      "source": [
        "**Download SNLI and QQP datasets:**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XoAnUhO4Gx_l",
        "colab_type": "code",
        "outputId": "05ae3bd9-08d5-40bf-a6e7-8aa7e95590ca",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "!cd multihead-siamese-nets/bin && ./prepare_data.sh"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2019-12-19 21:56:14--  https://drive.google.com/uc?export=download&id=1wkAjMu-Pqnm1l-92M7UEp5YEtT1cFgVz\n",
            "Resolving drive.google.com (drive.google.com)... 74.125.195.101, 74.125.195.139, 74.125.195.100, ...\n",
            "Connecting to drive.google.com (drive.google.com)|74.125.195.101|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Moved Temporarily\n",
            "Location: https://doc-04-8o-docs.googleusercontent.com/docs/securesc/ha0ro937gcuc7l7deffksulhg5h7mbp1/ojd0nn25bqu3aeocugcdv74nkte9gm6r/1576785600000/05563007606908372189/*/1wkAjMu-Pqnm1l-92M7UEp5YEtT1cFgVz?e=download [following]\n",
            "Warning: wildcards not supported in HTTP.\n",
            "--2019-12-19 21:56:16--  https://doc-04-8o-docs.googleusercontent.com/docs/securesc/ha0ro937gcuc7l7deffksulhg5h7mbp1/ojd0nn25bqu3aeocugcdv74nkte9gm6r/1576785600000/05563007606908372189/*/1wkAjMu-Pqnm1l-92M7UEp5YEtT1cFgVz?e=download\n",
            "Resolving doc-04-8o-docs.googleusercontent.com (doc-04-8o-docs.googleusercontent.com)... 74.125.197.132, 2607:f8b0:400e:c03::84\n",
            "Connecting to doc-04-8o-docs.googleusercontent.com (doc-04-8o-docs.googleusercontent.com)|74.125.197.132|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: unspecified [application/x-compressed-tar]\n",
            "Saving to: ‘SNLI/train_snli.tgz’\n",
            "\n",
            "SNLI/train_snli.tgz     [ <=>                ]   7.38M  --.-KB/s    in 0.1s    \n",
            "\n",
            "2019-12-19 21:56:16 (72.5 MB/s) - ‘SNLI/train_snli.tgz’ saved [7735242]\n",
            "\n",
            "--2019-12-19 21:56:16--  https://drive.google.com/uc?export=download&id=1dnck-CCIyx8y2xg1vwFzcwXieZJB7ERC\n",
            "Resolving drive.google.com (drive.google.com)... 74.125.142.138, 74.125.142.100, 74.125.142.101, ...\n",
            "Connecting to drive.google.com (drive.google.com)|74.125.142.138|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Moved Temporarily\n",
            "Location: https://doc-0o-8o-docs.googleusercontent.com/docs/securesc/ha0ro937gcuc7l7deffksulhg5h7mbp1/tova5um143uoa6menf6mf44iqt6q44vf/1576785600000/05563007606908372189/*/1dnck-CCIyx8y2xg1vwFzcwXieZJB7ERC?e=download [following]\n",
            "Warning: wildcards not supported in HTTP.\n",
            "--2019-12-19 21:56:20--  https://doc-0o-8o-docs.googleusercontent.com/docs/securesc/ha0ro937gcuc7l7deffksulhg5h7mbp1/tova5um143uoa6menf6mf44iqt6q44vf/1576785600000/05563007606908372189/*/1dnck-CCIyx8y2xg1vwFzcwXieZJB7ERC?e=download\n",
            "Resolving doc-0o-8o-docs.googleusercontent.com (doc-0o-8o-docs.googleusercontent.com)... 74.125.197.132, 2607:f8b0:400e:c03::84\n",
            "Connecting to doc-0o-8o-docs.googleusercontent.com (doc-0o-8o-docs.googleusercontent.com)|74.125.197.132|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: unspecified [application/x-gtar]\n",
            "Saving to: ‘QQP/qqp_train.tgz’\n",
            "\n",
            "QQP/qqp_train.tgz       [  <=>               ]  21.15M  69.5MB/s    in 0.3s    \n",
            "\n",
            "2019-12-19 21:56:20 (69.5 MB/s) - ‘QQP/qqp_train.tgz’ saved [22176174]\n",
            "\n",
            "--2019-12-19 21:56:20--  https://docs.google.com/uc?export=download&confirm=3r19&id=1XD-HxzUCTHrzhfvIXOlgqN_MWiiAqM8h\n",
            "Resolving docs.google.com (docs.google.com)... 172.253.117.139, 172.253.117.101, 172.253.117.102, ...\n",
            "Connecting to docs.google.com (docs.google.com)|172.253.117.139|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Moved Temporarily\n",
            "Location: https://doc-0g-8o-docs.googleusercontent.com/docs/securesc/ha0ro937gcuc7l7deffksulhg5h7mbp1/dutqrftmh3kbnm9no38c5468oqgb1r4l/1576785600000/05563007606908372189/*/1XD-HxzUCTHrzhfvIXOlgqN_MWiiAqM8h?e=download [following]\n",
            "Warning: wildcards not supported in HTTP.\n",
            "--2019-12-19 21:56:20--  https://doc-0g-8o-docs.googleusercontent.com/docs/securesc/ha0ro937gcuc7l7deffksulhg5h7mbp1/dutqrftmh3kbnm9no38c5468oqgb1r4l/1576785600000/05563007606908372189/*/1XD-HxzUCTHrzhfvIXOlgqN_MWiiAqM8h?e=download\n",
            "Resolving doc-0g-8o-docs.googleusercontent.com (doc-0g-8o-docs.googleusercontent.com)... 74.125.197.132, 2607:f8b0:400e:c03::84\n",
            "Connecting to doc-0g-8o-docs.googleusercontent.com (doc-0g-8o-docs.googleusercontent.com)|74.125.197.132|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: unspecified [application/x-gtar]\n",
            "Saving to: ‘QQP/qqp_test.tgz’\n",
            "\n",
            "QQP/qqp_test.tgz        [     <=>            ] 112.33M  54.0MB/s    in 2.1s    \n",
            "\n",
            "2019-12-19 21:56:23 (54.0 MB/s) - ‘QQP/qqp_test.tgz’ saved [117792766]\n",
            "\n",
            "--2019-12-19 21:56:23--  https://dl.fbaipublicfiles.com/anli/anli_v0.1.zip\n",
            "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 104.20.22.166, 104.20.6.166, 2606:4700:10::6814:6a6, ...\n",
            "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|104.20.22.166|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 18621352 (18M) [application/zip]\n",
            "Saving to: ‘ANLI/anli_v0.1.zip’\n",
            "\n",
            "ANLI/anli_v0.1.zip  100%[===================>]  17.76M  46.2MB/s    in 0.4s    \n",
            "\n",
            "2019-12-19 21:56:24 (46.2 MB/s) - ‘ANLI/anli_v0.1.zip’ saved [18621352/18621352]\n",
            "\n",
            "train_snli.txt\n",
            "train.csv\n",
            "test.csv\n",
            "Archive:  ANLI/anli_v0.1.zip\n",
            "   creating: ANLI/anli_v0.1/\n",
            "   creating: ANLI/anli_v0.1/R1/\n",
            "  inflating: ANLI/anli_v0.1/R1/train.jsonl  \n",
            "   creating: ANLI/__MACOSX/\n",
            "   creating: ANLI/__MACOSX/anli_v0.1/\n",
            "   creating: ANLI/__MACOSX/anli_v0.1/R1/\n",
            "  inflating: ANLI/__MACOSX/anli_v0.1/R1/._train.jsonl  \n",
            "  inflating: ANLI/anli_v0.1/R1/test.jsonl  \n",
            "  inflating: ANLI/__MACOSX/anli_v0.1/R1/._test.jsonl  \n",
            "  inflating: ANLI/anli_v0.1/R1/dev.jsonl  \n",
            "  inflating: ANLI/__MACOSX/anli_v0.1/R1/._dev.jsonl  \n",
            "  inflating: ANLI/__MACOSX/anli_v0.1/._R1  \n",
            "  inflating: ANLI/anli_v0.1/README.txt  \n",
            "  inflating: ANLI/__MACOSX/anli_v0.1/._README.txt  \n",
            "   creating: ANLI/anli_v0.1/R3/\n",
            "  inflating: ANLI/anli_v0.1/R3/train.jsonl  \n",
            "   creating: ANLI/__MACOSX/anli_v0.1/R3/\n",
            "  inflating: ANLI/__MACOSX/anli_v0.1/R3/._train.jsonl  \n",
            "  inflating: ANLI/anli_v0.1/R3/test.jsonl  \n",
            "  inflating: ANLI/__MACOSX/anli_v0.1/R3/._test.jsonl  \n",
            "  inflating: ANLI/anli_v0.1/R3/dev.jsonl  \n",
            "  inflating: ANLI/__MACOSX/anli_v0.1/R3/._dev.jsonl  \n",
            "  inflating: ANLI/__MACOSX/anli_v0.1/._R3  \n",
            "   creating: ANLI/anli_v0.1/R2/\n",
            "  inflating: ANLI/anli_v0.1/R2/train.jsonl  \n",
            "   creating: ANLI/__MACOSX/anli_v0.1/R2/\n",
            "  inflating: ANLI/__MACOSX/anli_v0.1/R2/._train.jsonl  \n",
            "  inflating: ANLI/anli_v0.1/R2/test.jsonl  \n",
            "  inflating: ANLI/__MACOSX/anli_v0.1/R2/._test.jsonl  \n",
            "  inflating: ANLI/anli_v0.1/R2/dev.jsonl  \n",
            "  inflating: ANLI/__MACOSX/anli_v0.1/R2/._dev.jsonl  \n",
            "  inflating: ANLI/__MACOSX/anli_v0.1/._R2  \n",
            "  inflating: ANLI/__MACOSX/._anli_v0.1  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9aJlYocoZEA0",
        "colab_type": "code",
        "outputId": "3dfa18cf-a5ef-4dcc-fa86-b655710ef6d9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        }
      },
      "source": [
        "!ls -l multihead-siamese-nets/corpora/"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 12\n",
            "drwxr-xr-x 4 root root 4096 Dec 19 21:56 ANLI\n",
            "drwxr-xr-x 2 root root 4096 Dec 19 21:56 QQP\n",
            "drwxr-xr-x 2 root root 4096 Dec 19 21:56 SNLI\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V9X_vpHXbWFu",
        "colab_type": "text"
      },
      "source": [
        "**Install all required dependecies:**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sxlvj6bo53gp",
        "colab_type": "code",
        "outputId": "d7aa383a-5ded-4d62-ad64-b525c6c16b60",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "!pip install -r multihead-siamese-nets/requirements/requirements-gpu.txt"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting tqdm==4.15.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4b/8c/f578581f1ea3e65af88a0c30c3d09d264b01acf585c1e6dcde022fbc664b/tqdm-4.15.0-py2.py3-none-any.whl (46kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 2.6MB/s \n",
            "\u001b[?25hCollecting pandas==0.22.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/da/c6/0936bc5814b429fddb5d6252566fe73a3e40372e6ceaf87de3dec1326f28/pandas-0.22.0-cp36-cp36m-manylinux1_x86_64.whl (26.2MB)\n",
            "\u001b[K     |████████████████████████████████| 26.3MB 116kB/s \n",
            "\u001b[?25hRequirement already satisfied: tflearn==0.3.2 in /usr/local/lib/python3.6/dist-packages (from -r multihead-siamese-nets/requirements/requirements-gpu.txt (line 3)) (0.3.2)\n",
            "Collecting numpy==1.14.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/6e/dc/92c0f670e7b986829fc92c4c0208edb9d72908149da38ecda50d816ea057/numpy-1.14.2-cp36-cp36m-manylinux1_x86_64.whl (12.2MB)\n",
            "\u001b[K     |████████████████████████████████| 12.2MB 47.7MB/s \n",
            "\u001b[?25hCollecting tensorflow-gpu==1.15.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a5/ad/933140e74973fb917a194ab814785e7c23680ca5dee6d663a509fe9579b6/tensorflow_gpu-1.15.0-cp36-cp36m-manylinux2010_x86_64.whl (411.5MB)\n",
            "\u001b[K     |████████████████████████████████| 411.5MB 40kB/s \n",
            "\u001b[?25hCollecting jsonlines==1.2.0\n",
            "  Downloading https://files.pythonhosted.org/packages/4f/9a/ab96291470e305504aa4b7a2e0ec132e930da89eb3ca7a82fbe03167c131/jsonlines-1.2.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied: seaborn==0.9.0 in /usr/local/lib/python3.6/dist-packages (from -r multihead-siamese-nets/requirements/requirements-gpu.txt (line 7)) (0.9.0)\n",
            "Requirement already satisfied: pytz>=2011k in /usr/local/lib/python3.6/dist-packages (from pandas==0.22.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 2)) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2 in /usr/local/lib/python3.6/dist-packages (from pandas==0.22.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 2)) (2.6.1)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.6/dist-packages (from tflearn==0.3.2->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 3)) (4.3.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from tflearn==0.3.2->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 3)) (1.12.0)\n",
            "Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.15.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 5)) (0.1.8)\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.15.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 5)) (1.1.0)\n",
            "Requirement already satisfied: tensorboard<1.16.0,>=1.15.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.15.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 5)) (1.15.0)\n",
            "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.15.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 5)) (3.10.0)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.15.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 5)) (1.11.2)\n",
            "Requirement already satisfied: gast==0.2.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.15.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 5)) (0.2.2)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.15.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 5)) (0.8.1)\n",
            "Requirement already satisfied: tensorflow-estimator==1.15.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.15.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 5)) (1.15.1)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.15.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 5)) (3.1.0)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.15.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 5)) (1.15.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.15.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 5)) (1.1.0)\n",
            "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.15.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 5)) (0.8.1)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.15.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 5)) (0.33.6)\n",
            "Requirement already satisfied: keras-applications>=1.0.8 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.15.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 5)) (1.0.8)\n",
            "Requirement already satisfied: scipy>=0.14.0 in /usr/local/lib/python3.6/dist-packages (from seaborn==0.9.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 7)) (1.3.3)\n",
            "Requirement already satisfied: matplotlib>=1.4.3 in /usr/local/lib/python3.6/dist-packages (from seaborn==0.9.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 7)) (3.1.2)\n",
            "Requirement already satisfied: olefile in /usr/local/lib/python3.6/dist-packages (from Pillow->tflearn==0.3.2->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 3)) (0.46)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow-gpu==1.15.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 5)) (3.1.1)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow-gpu==1.15.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 5)) (0.16.0)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow-gpu==1.15.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 5)) (42.0.2)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.8->tensorflow-gpu==1.15.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 5)) (2.8.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=1.4.3->seaborn==0.9.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 7)) (1.1.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=1.4.3->seaborn==0.9.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 7)) (0.10.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=1.4.3->seaborn==0.9.0->-r multihead-siamese-nets/requirements/requirements-gpu.txt (line 7)) (2.4.5)\n",
            "\u001b[31mERROR: tensorflow 1.15.0 has requirement numpy<2.0,>=1.16.0, but you'll have numpy 1.14.2 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: spacy 2.1.9 has requirement numpy>=1.15.0, but you'll have numpy 1.14.2 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: plotnine 0.5.1 has requirement pandas>=0.23.4, but you'll have pandas 0.22.0 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: mizani 0.5.4 has requirement pandas>=0.23.4, but you'll have pandas 0.22.0 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: imgaug 0.2.9 has requirement numpy>=1.15.0, but you'll have numpy 1.14.2 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: google-colab 1.0.0 has requirement pandas~=0.25.0; python_version >= \"3.0\", but you'll have pandas 0.22.0 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: featuretools 0.4.1 has requirement pandas>=0.23.0, but you'll have pandas 0.22.0 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: featuretools 0.4.1 has requirement tqdm>=4.19.2, but you'll have tqdm 4.15.0 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: fbprophet 0.5 has requirement pandas>=0.23.4, but you'll have pandas 0.22.0 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: fastai 1.0.59 has requirement numpy>=1.15, but you'll have numpy 1.14.2 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: cvxpy 1.0.25 has requirement numpy>=1.15, but you'll have numpy 1.14.2 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: blis 0.2.4 has requirement numpy>=1.15.0, but you'll have numpy 1.14.2 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.9 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: tensorflow-gpu 1.15.0 has requirement numpy<2.0,>=1.16.0, but you'll have numpy 1.14.2 which is incompatible.\u001b[0m\n",
            "Installing collected packages: tqdm, numpy, pandas, tensorflow-gpu, jsonlines\n",
            "  Found existing installation: tqdm 4.28.1\n",
            "    Uninstalling tqdm-4.28.1:\n",
            "      Successfully uninstalled tqdm-4.28.1\n",
            "  Found existing installation: numpy 1.17.4\n",
            "    Uninstalling numpy-1.17.4:\n",
            "      Successfully uninstalled numpy-1.17.4\n",
            "  Found existing installation: pandas 0.25.3\n",
            "    Uninstalling pandas-0.25.3:\n",
            "      Successfully uninstalled pandas-0.25.3\n",
            "Successfully installed jsonlines-1.2.0 numpy-1.14.2 pandas-0.22.0 tensorflow-gpu-1.15.0 tqdm-4.15.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "numpy",
                  "pandas",
                  "tqdm"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ic7wgrH6I-IC",
        "colab_type": "code",
        "outputId": "67fdc7d3-429e-4d23-e726-3d692ba3b86a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 250
        }
      },
      "source": [
        "%%shell\n",
        "wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip\n",
        "unzip ngrok-stable-linux-amd64.zip"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2019-04-18 20:09:09--  https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip\n",
            "Resolving bin.equinox.io (bin.equinox.io)... 52.3.53.115, 34.206.9.96, 54.152.127.232, ...\n",
            "Connecting to bin.equinox.io (bin.equinox.io)|52.3.53.115|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 14977695 (14M) [application/octet-stream]\n",
            "Saving to: ‘ngrok-stable-linux-amd64.zip’\n",
            "\n",
            "ngrok-stable-linux- 100%[===================>]  14.28M  8.44MB/s    in 1.7s    \n",
            "\n",
            "2019-04-18 20:09:12 (8.44 MB/s) - ‘ngrok-stable-linux-amd64.zip’ saved [14977695/14977695]\n",
            "\n",
            "Archive:  ngrok-stable-linux-amd64.zip\n",
            "  inflating: ngrok                   \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              ""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aDsUZWMRc3vZ",
        "colab_type": "text"
      },
      "source": [
        "**Run TensorBoard for visualization**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u1IYZ9iZT_VI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "LOG_DIR = './logs'\n",
        "get_ipython().system_raw(\n",
        "    'tensorboard --logdir {} --host 0.0.0.0 --port 6006 &'\n",
        "    .format(LOG_DIR)\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OM_xcDjACT_w",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "get_ipython().system_raw('./ngrok http 6006 &')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8ZQiWRqBCYhU",
        "colab_type": "code",
        "outputId": "9fcc5a72-2e52-4ee6-b90c-34994e8fa2d4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "!curl -s http://localhost:4040/api/tunnels | python3 -c \\\n",
        "    \"import sys, json; print(json.load(sys.stdin)['tunnels'][0]['public_url'])\""
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "https://3bacb966.ngrok.io\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BBJWFg5tK_02",
        "colab_type": "text"
      },
      "source": [
        "**Prepare configuaration files**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vYGQ79Wv9yfF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!mkdir config\n",
        "!cp multihead-siamese-nets/config/main.ini config\n",
        "!cp -r multihead-siamese-nets/config/model config\n",
        "!cp -r multihead-siamese-nets/corpora ."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Em2q5LYPLIjJ",
        "colab_type": "text"
      },
      "source": [
        "**Train CNN, RNN and MULTIHEAD models on SNLI corpus**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-d1OX3EebRtA",
        "colab_type": "code",
        "outputId": "cc51051e-1c33-4558-a916-d08d44a4e5d2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "!cd multihead-siamese-nets && python3 run.py train cnn SNLI"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tflearn/helpers/summarizer.py:9: The name tf.summary.merge is deprecated. Please use tf.compat.v1.summary.merge instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tflearn/helpers/trainer.py:25: The name tf.summary.FileWriter is deprecated. Please use tf.compat.v1.summary.FileWriter instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tflearn/collections.py:13: The name tf.GraphKeys is deprecated. Please use tf.compat.v1.GraphKeys instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tflearn/config.py:123: The name tf.get_collection is deprecated. Please use tf.compat.v1.get_collection instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tflearn/config.py:129: The name tf.add_to_collection is deprecated. Please use tf.compat.v1.add_to_collection instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tflearn/config.py:131: The name tf.assign is deprecated. Please use tf.compat.v1.assign instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/utils/data_utils.py:7: The name tf.logging.info is deprecated. Please use tf.compat.v1.logging.info instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/utils/other_utils.py:10: The name tf.logging.set_verbosity is deprecated. Please use tf.compat.v1.logging.set_verbosity instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/utils/other_utils.py:10: The name tf.logging.INFO is deprecated. Please use tf.compat.v1.logging.INFO instead.\n",
            "\n",
            "INFO:tensorflow:Setting visible GPU to 0\n",
            "INFO:tensorflow:Reading main configuration.\n",
            "INFO:tensorflow:Reading configuration for cnn model.\n",
            "INFO:tensorflow:Chosen word embeddings.\n",
            "INFO:tensorflow:Maximum sentence length : 78\n",
            "INFO:tensorflow:Processing sentences with word embeddings...\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tflearn/data_utils.py:201: VocabularyProcessor.__init__ (from tensorflow.contrib.learn.python.learn.preprocessing.text) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tensorflow/transform or tf.data.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/contrib/learn/python/learn/preprocessing/text.py:154: CategoricalVocabulary.__init__ (from tensorflow.contrib.learn.python.learn.preprocessing.categorical_vocabulary) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tensorflow/transform or tf.data.\n",
            "INFO:tensorflow:Sentences have been successfully processed.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/contrib/learn/python/learn/preprocessing/text.py:170: tokenizer (from tensorflow.contrib.learn.python.learn.preprocessing.text) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tensorflow/transform or tf.data.\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/models/base_model.py:15: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/models/base_model.py:28: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/models/base_model.py:29: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/layers/convolution.py:21: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.keras.layers.Conv2D` instead.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `layer.__call__` method instead.\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/layers/convolution.py:27: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use keras.layers.MaxPooling2D instead.\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/layers/basics.py:32: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use keras.layers.dropout instead.\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/layers/losses.py:55: The name tf.losses.mean_squared_error is deprecated. Please use tf.compat.v1.losses.mean_squared_error instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/losses/losses_impl.py:121: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/layers/basics.py:66: The name tf.train.AdamOptimizer is deprecated. Please use tf.compat.v1.train.AdamOptimizer instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/models/base_model.py:42: The name tf.rint is deprecated. Please use tf.math.rint instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/models/base_model.py:43: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.cast` instead.\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/models/base_model.py:47: The name tf.summary.scalar is deprecated. Please use tf.compat.v1.summary.scalar instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/models/base_model.py:49: The name tf.summary.merge_all is deprecated. Please use tf.compat.v1.summary.merge_all instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/utils/model_saver.py:9: The name tf.train.Saver is deprecated. Please use tf.compat.v1.train.Saver instead.\n",
            "\n",
            "WARNING:tensorflow:From run.py:73: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
            "\n",
            "WARNING:tensorflow:From run.py:78: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
            "\n",
            "2019-12-19 21:58:20.199009: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
            "2019-12-19 21:58:20.203484: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2200000000 Hz\n",
            "2019-12-19 21:58:20.203685: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x24c5480 initialized for platform Host (this does not guarantee that XLA will be used). Devices:\n",
            "2019-12-19 21:58:20.203714: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version\n",
            "2019-12-19 21:58:20.225354: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1\n",
            "2019-12-19 21:58:20.371567: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2019-12-19 21:58:20.372223: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x24c52c0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
            "2019-12-19 21:58:20.372248: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Tesla P100-PCIE-16GB, Compute Capability 6.0\n",
            "2019-12-19 21:58:20.372551: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2019-12-19 21:58:20.373108: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 0 with properties: \n",
            "name: Tesla P100-PCIE-16GB major: 6 minor: 0 memoryClockRate(GHz): 1.3285\n",
            "pciBusID: 0000:00:04.0\n",
            "2019-12-19 21:58:20.386202: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0\n",
            "2019-12-19 21:58:20.560055: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0\n",
            "2019-12-19 21:58:20.654600: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10.0\n",
            "2019-12-19 21:58:20.677003: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10.0\n",
            "2019-12-19 21:58:20.906854: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10.0\n",
            "2019-12-19 21:58:21.053975: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10.0\n",
            "2019-12-19 21:58:21.555134: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7\n",
            "2019-12-19 21:58:21.555313: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2019-12-19 21:58:21.556012: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2019-12-19 21:58:21.556551: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Adding visible gpu devices: 0\n",
            "2019-12-19 21:58:21.556619: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0\n",
            "2019-12-19 21:58:21.558018: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1159] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
            "2019-12-19 21:58:21.558065: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1165]      0 \n",
            "2019-12-19 21:58:21.558076: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1178] 0:   N \n",
            "2019-12-19 21:58:21.558182: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2019-12-19 21:58:21.558801: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2019-12-19 21:58:21.559370: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:39] Overriding allow_growth setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\n",
            "2019-12-19 21:58:21.559441: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1304] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 15216 MB memory) -> physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0)\n",
            "WARNING:tensorflow:From run.py:80: The name tf.global_variables_initializer is deprecated. Please use tf.compat.v1.global_variables_initializer instead.\n",
            "\n",
            "INFO:tensorflow:Training model for 10 epochs\n",
            "Epochs:   0% 0/10 [00:00<?, ?it/s]\n",
            "Batches:   0% 0/638 [00:00<?, ?it/s, acc=0]2019-12-19 21:58:22.843702: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7\n",
            "2019-12-19 21:58:27.358433: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0\n",
            "\u001b[A\n",
            "Batches:   0% 1/638 [00:06<1:07:35,  6.37s/it, dev_acc=0.49, epoch=0, loss=0.39, train_acc=0.51]\u001b[A\n",
            "Batches:   1% 7/638 [00:06<46:55,  4.46s/it, dev_acc=0.49, epoch=0, loss=0.39, train_acc=0.51]  \u001b[A\n",
            "Batches:   2% 13/638 [00:06<32:35,  3.13s/it, dev_acc=0.49, epoch=0, loss=0.39, train_acc=0.51]\u001b[A\n",
            "Batches:   3% 19/638 [00:06<22:39,  2.20s/it, dev_acc=0.49, epoch=0, loss=0.39, train_acc=0.51]\u001b[A\n",
            "Batches:   4% 23/638 [00:06<15:51,  1.55s/it, dev_acc=0.58, epoch=0, loss=0.24, train_acc=0.57]\u001b[A\n",
            "Batches:   5% 29/638 [00:06<11:03,  1.09s/it, dev_acc=0.58, epoch=0, loss=0.24, train_acc=0.57]\u001b[A\n",
            "Batches:   5% 35/638 [00:07<07:43,  1.30it/s, dev_acc=0.58, epoch=0, loss=0.24, train_acc=0.57]\u001b[A\n",
            "Batches:   6% 41/638 [00:07<05:25,  1.83it/s, dev_acc=0.66, epoch=0, loss=0.23, train_acc=0.64]\u001b[A\n",
            "Batches:   7% 47/638 [00:07<03:49,  2.58it/s, dev_acc=0.66, epoch=0, loss=0.23, train_acc=0.64]\u001b[A\n",
            "Batches:   8% 53/638 [00:07<02:42,  3.60it/s, dev_acc=0.66, epoch=0, loss=0.23, train_acc=0.64]\u001b[A\n",
            "Batches:   9% 59/638 [00:07<01:55,  5.00it/s, dev_acc=0.66, epoch=0, loss=0.23, train_acc=0.64]\u001b[A\n",
            "Batches:  10% 64/638 [00:07<01:25,  6.71it/s, dev_acc=0.69, epoch=0, loss=0.22, train_acc=0.67]\u001b[A\n",
            "Batches:  11% 70/638 [00:07<01:02,  9.09it/s, dev_acc=0.69, epoch=0, loss=0.22, train_acc=0.67]\u001b[A\n",
            "Batches:  12% 76/638 [00:07<00:46, 12.07it/s, dev_acc=0.69, epoch=0, loss=0.22, train_acc=0.67]\u001b[A\n",
            "Batches:  13% 81/638 [00:08<00:37, 14.93it/s, dev_acc=0.73, epoch=0, loss=0.21, train_acc=0.71]\u001b[A\n",
            "Batches:  14% 87/638 [00:08<00:29, 18.97it/s, dev_acc=0.73, epoch=0, loss=0.21, train_acc=0.71]\u001b[A\n",
            "Batches:  15% 93/638 [00:08<00:23, 23.45it/s, dev_acc=0.73, epoch=0, loss=0.21, train_acc=0.71]\u001b[A\n",
            "Batches:  16% 99/638 [00:08<00:19, 27.99it/s, dev_acc=0.73, epoch=0, loss=0.21, train_acc=0.71]\u001b[A\n",
            "Batches:  16% 104/638 [00:08<00:22, 24.19it/s, dev_acc=0.75, epoch=0, loss=0.19, train_acc=0.73]\u001b[A\n",
            "Batches:  17% 110/638 [00:08<00:18, 28.75it/s, dev_acc=0.75, epoch=0, loss=0.19, train_acc=0.73]\u001b[A\n",
            "Batches:  18% 116/638 [00:08<00:15, 33.25it/s, dev_acc=0.75, epoch=0, loss=0.19, train_acc=0.73]\u001b[A\n",
            "Batches:  19% 121/638 [00:09<00:15, 33.26it/s, dev_acc=0.75, epoch=0, loss=0.19, train_acc=0.73]\u001b[A\n",
            "Batches:  20% 127/638 [00:09<00:13, 37.17it/s, dev_acc=0.75, epoch=0, loss=0.19, train_acc=0.73]\u001b[A\n",
            "Batches:  21% 133/638 [00:09<00:12, 40.47it/s, dev_acc=0.75, epoch=0, loss=0.19, train_acc=0.73]\u001b[A\n",
            "Batches:  22% 139/638 [00:09<00:11, 43.42it/s, dev_acc=0.75, epoch=0, loss=0.19, train_acc=0.73]\u001b[A\n",
            "Batches:  23% 144/638 [00:09<00:12, 40.46it/s, dev_acc=0.75, epoch=0, loss=0.16, train_acc=0.75]\u001b[A\n",
            "Batches:  24% 150/638 [00:09<00:11, 43.55it/s, dev_acc=0.75, epoch=0, loss=0.16, train_acc=0.75]\u001b[A\n",
            "Batches:  24% 156/638 [00:09<00:10, 46.14it/s, dev_acc=0.75, epoch=0, loss=0.16, train_acc=0.75]\u001b[A\n",
            "Batches:  25% 161/638 [00:09<00:11, 42.03it/s, dev_acc=0.76, epoch=0, loss=0.16, train_acc=0.75]\u001b[A\n",
            "Batches:  26% 167/638 [00:10<00:10, 44.93it/s, dev_acc=0.76, epoch=0, loss=0.16, train_acc=0.75]\u001b[A\n",
            "Batches:  27% 173/638 [00:10<00:09, 46.59it/s, dev_acc=0.76, epoch=0, loss=0.16, train_acc=0.75]\u001b[A\n",
            "Batches:  28% 179/638 [00:10<00:09, 48.44it/s, dev_acc=0.76, epoch=0, loss=0.16, train_acc=0.75]\u001b[A\n",
            "Batches:  29% 184/638 [00:10<00:10, 42.31it/s, dev_acc=0.76, epoch=0, loss=0.17, train_acc=0.76]\u001b[A\n",
            "Batches:  30% 190/638 [00:10<00:09, 45.90it/s, dev_acc=0.76, epoch=0, loss=0.17, train_acc=0.76]\u001b[A\n",
            "Batches:  31% 196/638 [00:10<00:09, 47.67it/s, dev_acc=0.76, epoch=0, loss=0.17, train_acc=0.76]\u001b[A\n",
            "Batches:  32% 201/638 [00:10<00:11, 36.47it/s, dev_acc=0.76, epoch=0, loss=0.18, train_acc=0.76]\u001b[A\n",
            "Batches:  32% 207/638 [00:11<00:10, 40.22it/s, dev_acc=0.76, epoch=0, loss=0.18, train_acc=0.76]\u001b[A\n",
            "Batches:  33% 213/638 [00:11<00:09, 43.35it/s, dev_acc=0.76, epoch=0, loss=0.18, train_acc=0.76]\u001b[A\n",
            "Batches:  34% 219/638 [00:11<00:09, 45.70it/s, dev_acc=0.76, epoch=0, loss=0.18, train_acc=0.76]\u001b[A\n",
            "Batches:  35% 224/638 [00:11<00:10, 40.21it/s, dev_acc=0.76, epoch=0, loss=0.16, train_acc=0.76]\u001b[A\n",
            "Batches:  36% 230/638 [00:11<00:09, 43.48it/s, dev_acc=0.76, epoch=0, loss=0.16, train_acc=0.76]\u001b[A\n",
            "Batches:  37% 236/638 [00:11<00:08, 45.63it/s, dev_acc=0.76, epoch=0, loss=0.16, train_acc=0.76]\u001b[A\n",
            "Batches:  38% 241/638 [00:11<00:09, 41.95it/s, dev_acc=0.76, epoch=0, loss=0.17, train_acc=0.76]\u001b[A\n",
            "Batches:  39% 247/638 [00:11<00:08, 45.21it/s, dev_acc=0.76, epoch=0, loss=0.17, train_acc=0.76]\u001b[A\n",
            "Batches:  40% 253/638 [00:12<00:08, 47.84it/s, dev_acc=0.76, epoch=0, loss=0.17, train_acc=0.76]\u001b[A\n",
            "Batches:  40% 258/638 [00:12<00:07, 48.40it/s, dev_acc=0.76, epoch=0, loss=0.17, train_acc=0.76]\u001b[A\n",
            "Batches:  41% 263/638 [00:12<00:08, 43.10it/s, dev_acc=0.77, epoch=0, loss=0.17, train_acc=0.77]\u001b[A\n",
            "Batches:  42% 269/638 [00:12<00:08, 45.35it/s, dev_acc=0.77, epoch=0, loss=0.17, train_acc=0.77]\u001b[A\n",
            "Batches:  43% 275/638 [00:12<00:07, 47.82it/s, dev_acc=0.77, epoch=0, loss=0.17, train_acc=0.77]\u001b[A\n",
            "Batches:  44% 281/638 [00:12<00:08, 43.67it/s, dev_acc=0.76, epoch=0, loss=0.18, train_acc=0.78]\u001b[A\n",
            "Batches:  45% 287/638 [00:12<00:07, 45.91it/s, dev_acc=0.76, epoch=0, loss=0.18, train_acc=0.78]\u001b[A\n",
            "Batches:  46% 293/638 [00:12<00:07, 47.65it/s, dev_acc=0.76, epoch=0, loss=0.18, train_acc=0.78]\u001b[A\n",
            "Batches:  47% 299/638 [00:12<00:06, 49.41it/s, dev_acc=0.76, epoch=0, loss=0.18, train_acc=0.78]\u001b[A\n",
            "Batches:  48% 305/638 [00:13<00:08, 38.24it/s, dev_acc=0.77, epoch=0, loss=0.16, train_acc=0.78]\u001b[A\n",
            "Batches:  49% 311/638 [00:13<00:07, 41.57it/s, dev_acc=0.77, epoch=0, loss=0.16, train_acc=0.78]\u001b[A\n",
            "Batches:  50% 317/638 [00:13<00:07, 44.12it/s, dev_acc=0.77, epoch=0, loss=0.16, train_acc=0.78]\u001b[A\n",
            "Batches:  50% 322/638 [00:13<00:07, 40.46it/s, dev_acc=0.76, epoch=0, loss=0.16, train_acc=0.78]\u001b[A\n",
            "Batches:  51% 328/638 [00:13<00:07, 43.85it/s, dev_acc=0.76, epoch=0, loss=0.16, train_acc=0.78]\u001b[A\n",
            "Batches:  52% 334/638 [00:13<00:06, 46.91it/s, dev_acc=0.76, epoch=0, loss=0.16, train_acc=0.78]\u001b[A\n",
            "Batches:  53% 340/638 [00:13<00:06, 49.16it/s, dev_acc=0.76, epoch=0, loss=0.16, train_acc=0.78]\u001b[A\n",
            "Batches:  54% 346/638 [00:14<00:06, 44.62it/s, dev_acc=0.77, epoch=0, loss=0.17, train_acc=0.78]\u001b[A\n",
            "Batches:  55% 351/638 [00:14<00:06, 45.34it/s, dev_acc=0.77, epoch=0, loss=0.17, train_acc=0.78]\u001b[A\n",
            "Batches:  56% 357/638 [00:14<00:05, 47.42it/s, dev_acc=0.77, epoch=0, loss=0.17, train_acc=0.78]\u001b[A\n",
            "Batches:  57% 362/638 [00:14<00:06, 42.43it/s, dev_acc=0.77, epoch=0, loss=0.16, train_acc=0.78]\u001b[A\n",
            "Batches:  58% 368/638 [00:14<00:06, 44.71it/s, dev_acc=0.77, epoch=0, loss=0.16, train_acc=0.78]\u001b[A\n",
            "Batches:  59% 374/638 [00:14<00:05, 46.54it/s, dev_acc=0.77, epoch=0, loss=0.16, train_acc=0.78]\u001b[A\n",
            "Batches:  60% 380/638 [00:14<00:05, 48.13it/s, dev_acc=0.77, epoch=0, loss=0.16, train_acc=0.78]\u001b[A\n",
            "Batches:  60% 385/638 [00:14<00:06, 42.03it/s, dev_acc=0.77, epoch=0, loss=0.17, train_acc=0.79]\u001b[A\n",
            "Batches:  61% 391/638 [00:15<00:05, 44.57it/s, dev_acc=0.77, epoch=0, loss=0.17, train_acc=0.79]\u001b[A\n",
            "Batches:  62% 397/638 [00:15<00:05, 46.95it/s, dev_acc=0.77, epoch=0, loss=0.17, train_acc=0.79]\u001b[A\n",
            "Batches:  63% 402/638 [00:15<00:07, 32.14it/s, dev_acc=0.77, epoch=0, loss=0.18, train_acc=0.79]\u001b[A\n",
            "Batches:  64% 408/638 [00:15<00:06, 36.11it/s, dev_acc=0.77, epoch=0, loss=0.18, train_acc=0.79]\u001b[A\n",
            "Batches:  65% 414/638 [00:15<00:05, 39.75it/s, dev_acc=0.77, epoch=0, loss=0.18, train_acc=0.79]\u001b[A\n",
            "Batches:  66% 419/638 [00:15<00:05, 42.14it/s, dev_acc=0.77, epoch=0, loss=0.18, train_acc=0.79]\u001b[A\n",
            "Batches:  66% 424/638 [00:15<00:05, 39.38it/s, dev_acc=0.78, epoch=0, loss=0.17, train_acc=0.79]\u001b[A\n",
            "Batches:  67% 430/638 [00:16<00:04, 42.24it/s, dev_acc=0.78, epoch=0, loss=0.17, train_acc=0.79]\u001b[A\n",
            "Batches:  68% 436/638 [00:16<00:04, 44.64it/s, dev_acc=0.78, epoch=0, loss=0.17, train_acc=0.79]\u001b[A\n",
            "Batches:  69% 441/638 [00:16<00:04, 40.80it/s, dev_acc=0.78, epoch=0, loss=0.16, train_acc=0.79]\u001b[A\n",
            "Batches:  70% 447/638 [00:16<00:04, 44.14it/s, dev_acc=0.78, epoch=0, loss=0.16, train_acc=0.79]\u001b[A\n",
            "Batches:  71% 453/638 [00:16<00:03, 46.38it/s, dev_acc=0.78, epoch=0, loss=0.16, train_acc=0.79]\u001b[A\n",
            "Batches:  72% 459/638 [00:16<00:03, 47.90it/s, dev_acc=0.78, epoch=0, loss=0.16, train_acc=0.79]\u001b[A\n",
            "Batches:  73% 464/638 [00:16<00:04, 42.87it/s, dev_acc=0.78, epoch=0, loss=0.16, train_acc=0.80]\u001b[A\n",
            "Batches:  74% 470/638 [00:16<00:03, 45.52it/s, dev_acc=0.78, epoch=0, loss=0.16, train_acc=0.80]\u001b[A\n",
            "Batches:  75% 476/638 [00:17<00:03, 47.93it/s, dev_acc=0.78, epoch=0, loss=0.16, train_acc=0.80]\u001b[A\n",
            "Batches:  75% 481/638 [00:17<00:03, 42.80it/s, dev_acc=0.78, epoch=0, loss=0.17, train_acc=0.80]\u001b[A\n",
            "Batches:  76% 487/638 [00:17<00:03, 45.55it/s, dev_acc=0.78, epoch=0, loss=0.17, train_acc=0.80]\u001b[A\n",
            "Batches:  77% 493/638 [00:17<00:03, 47.92it/s, dev_acc=0.78, epoch=0, loss=0.17, train_acc=0.80]\u001b[A\n",
            "Batches:  78% 499/638 [00:17<00:02, 49.02it/s, dev_acc=0.78, epoch=0, loss=0.17, train_acc=0.80]\u001b[A\n",
            "Batches:  79% 505/638 [00:17<00:03, 37.82it/s, dev_acc=0.79, epoch=0, loss=0.17, train_acc=0.80]\u001b[A\n",
            "Batches:  80% 511/638 [00:17<00:03, 41.25it/s, dev_acc=0.79, epoch=0, loss=0.17, train_acc=0.80]\u001b[A\n",
            "Batches:  81% 517/638 [00:17<00:02, 43.93it/s, dev_acc=0.79, epoch=0, loss=0.17, train_acc=0.80]\u001b[A\n",
            "Batches:  82% 522/638 [00:18<00:02, 40.71it/s, dev_acc=0.78, epoch=0, loss=0.16, train_acc=0.80]\u001b[A\n",
            "Batches:  83% 528/638 [00:18<00:02, 43.31it/s, dev_acc=0.78, epoch=0, loss=0.16, train_acc=0.80]\u001b[A\n",
            "Batches:  84% 534/638 [00:18<00:02, 45.30it/s, dev_acc=0.78, epoch=0, loss=0.16, train_acc=0.80]\u001b[A\n",
            "Batches:  85% 540/638 [00:18<00:02, 47.08it/s, dev_acc=0.78, epoch=0, loss=0.16, train_acc=0.80]\u001b[A\n",
            "Batches:  85% 545/638 [00:18<00:02, 41.49it/s, dev_acc=0.78, epoch=0, loss=0.17, train_acc=0.80]\u001b[A\n",
            "Batches:  86% 551/638 [00:18<00:01, 44.29it/s, dev_acc=0.78, epoch=0, loss=0.17, train_acc=0.80]\u001b[A\n",
            "Batches:  87% 557/638 [00:18<00:01, 46.50it/s, dev_acc=0.78, epoch=0, loss=0.17, train_acc=0.80]\u001b[A\n",
            "Batches:  88% 562/638 [00:19<00:01, 42.03it/s, dev_acc=0.78, epoch=0, loss=0.16, train_acc=0.80]\u001b[A\n",
            "Batches:  89% 568/638 [00:19<00:01, 45.07it/s, dev_acc=0.78, epoch=0, loss=0.16, train_acc=0.80]\u001b[A\n",
            "Batches:  90% 574/638 [00:19<00:01, 47.08it/s, dev_acc=0.78, epoch=0, loss=0.16, train_acc=0.80]\u001b[A\n",
            "Batches:  91% 580/638 [00:19<00:01, 48.48it/s, dev_acc=0.78, epoch=0, loss=0.16, train_acc=0.80]\u001b[A\n",
            "Batches:  92% 585/638 [00:19<00:01, 43.19it/s, dev_acc=0.78, epoch=0, loss=0.15, train_acc=0.80]\u001b[A\n",
            "Batches:  93% 591/638 [00:19<00:01, 45.48it/s, dev_acc=0.78, epoch=0, loss=0.15, train_acc=0.80]\u001b[A\n",
            "Batches:  94% 597/638 [00:19<00:00, 47.33it/s, dev_acc=0.78, epoch=0, loss=0.15, train_acc=0.80]\u001b[AWARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/training/saver.py:963: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use standard file APIs to delete files with this prefix.\n",
            "\n",
            "Batches:  94% 602/638 [00:19<00:01, 35.48it/s, dev_acc=0.78, epoch=0, loss=0.16, train_acc=0.80]\u001b[A\n",
            "Batches:  95% 608/638 [00:20<00:00, 39.40it/s, dev_acc=0.78, epoch=0, loss=0.16, train_acc=0.80]\u001b[A\n",
            "Batches:  96% 614/638 [00:20<00:00, 42.82it/s, dev_acc=0.78, epoch=0, loss=0.16, train_acc=0.80]\u001b[A\n",
            "Batches:  97% 620/638 [00:20<00:00, 45.92it/s, dev_acc=0.78, epoch=0, loss=0.16, train_acc=0.80]\u001b[A\n",
            "Batches:  98% 625/638 [00:20<00:00, 41.32it/s, dev_acc=0.78, epoch=0, loss=0.16, train_acc=0.80]\u001b[A\n",
            "Batches:  99% 631/638 [00:20<00:00, 44.09it/s, dev_acc=0.78, epoch=0, loss=0.16, train_acc=0.80]\u001b[A\n",
            "Batches: 100% 636/638 [00:20<00:00, 44.05it/s, dev_acc=0.78, epoch=0, loss=0.16, train_acc=0.80]\u001b[A\n",
            "Epochs:  10% 1/10 [00:21<03:11, 21.29s/it]\n",
            "Batches:   0% 0/638 [00:00<?, ?it/s, acc=0]\u001b[A\n",
            "Batches:   0% 2/638 [00:00<00:32, 19.37it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:   1% 8/638 [00:00<00:26, 24.00it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:   2% 14/638 [00:00<00:21, 28.74it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:   3% 20/638 [00:00<00:18, 33.39it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:   4% 24/638 [00:00<00:18, 32.81it/s, dev_acc=0.78, epoch=1, loss=0.16, train_acc=0.81]\u001b[A\n",
            "Batches:   5% 30/638 [00:00<00:16, 37.08it/s, dev_acc=0.78, epoch=1, loss=0.16, train_acc=0.81]\u001b[A\n",
            "Batches:   6% 36/638 [00:00<00:14, 41.02it/s, dev_acc=0.78, epoch=1, loss=0.16, train_acc=0.81]\u001b[A\n",
            "Batches:   6% 41/638 [00:00<00:15, 38.53it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:   7% 47/638 [00:01<00:13, 42.55it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:   8% 53/638 [00:01<00:12, 45.46it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:   9% 59/638 [00:01<00:12, 47.18it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:  10% 64/638 [00:01<00:16, 35.43it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:  11% 69/638 [00:01<00:14, 38.75it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:  12% 75/638 [00:01<00:13, 41.81it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:  13% 81/638 [00:01<00:13, 40.07it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  14% 87/638 [00:01<00:12, 43.53it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  14% 92/638 [00:02<00:12, 45.19it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  15% 98/638 [00:02<00:11, 47.36it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  16% 103/638 [00:02<00:12, 42.20it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  17% 109/638 [00:02<00:12, 43.85it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  18% 115/638 [00:02<00:11, 45.94it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  19% 121/638 [00:02<00:12, 42.54it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:  20% 127/638 [00:02<00:11, 45.23it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:  21% 133/638 [00:02<00:10, 47.95it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:  22% 139/638 [00:03<00:10, 49.55it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:  23% 145/638 [00:03<00:10, 45.16it/s, dev_acc=0.78, epoch=1, loss=0.16, train_acc=0.81]\u001b[A\n",
            "Batches:  24% 151/638 [00:03<00:10, 47.28it/s, dev_acc=0.78, epoch=1, loss=0.16, train_acc=0.81]\u001b[A\n",
            "Batches:  25% 157/638 [00:03<00:09, 48.83it/s, dev_acc=0.78, epoch=1, loss=0.16, train_acc=0.81]\u001b[A\n",
            "Batches:  25% 162/638 [00:03<00:13, 36.48it/s, dev_acc=0.78, epoch=1, loss=0.14, train_acc=0.80]\u001b[A\n",
            "Batches:  26% 168/638 [00:03<00:11, 40.44it/s, dev_acc=0.78, epoch=1, loss=0.14, train_acc=0.80]\u001b[A\n",
            "Batches:  27% 174/638 [00:03<00:10, 43.08it/s, dev_acc=0.78, epoch=1, loss=0.14, train_acc=0.80]\u001b[A\n",
            "Batches:  28% 180/638 [00:04<00:10, 45.20it/s, dev_acc=0.78, epoch=1, loss=0.14, train_acc=0.80]\u001b[A\n",
            "Batches:  29% 185/638 [00:04<00:10, 41.60it/s, dev_acc=0.78, epoch=1, loss=0.14, train_acc=0.80]\u001b[A\n",
            "Batches:  30% 191/638 [00:04<00:10, 43.92it/s, dev_acc=0.78, epoch=1, loss=0.14, train_acc=0.80]\u001b[A\n",
            "Batches:  31% 197/638 [00:04<00:09, 45.90it/s, dev_acc=0.78, epoch=1, loss=0.14, train_acc=0.80]\u001b[A\n",
            "Batches:  32% 202/638 [00:04<00:10, 41.66it/s, dev_acc=0.78, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  33% 208/638 [00:04<00:09, 44.47it/s, dev_acc=0.78, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  34% 214/638 [00:04<00:09, 46.50it/s, dev_acc=0.78, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  34% 220/638 [00:04<00:08, 48.20it/s, dev_acc=0.78, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  35% 225/638 [00:05<00:09, 42.54it/s, dev_acc=0.78, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  36% 231/638 [00:05<00:09, 45.21it/s, dev_acc=0.78, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  37% 237/638 [00:05<00:08, 46.80it/s, dev_acc=0.78, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  38% 242/638 [00:05<00:09, 41.90it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  39% 248/638 [00:05<00:08, 44.24it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  40% 254/638 [00:05<00:08, 46.25it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  41% 260/638 [00:05<00:07, 47.70it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  42% 265/638 [00:06<00:10, 35.30it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:  42% 271/638 [00:06<00:09, 39.41it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:  43% 277/638 [00:06<00:08, 42.38it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:  44% 282/638 [00:06<00:09, 39.51it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.82]\u001b[A\n",
            "Batches:  45% 288/638 [00:06<00:08, 43.40it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.82]\u001b[A\n",
            "Batches:  46% 294/638 [00:06<00:07, 45.82it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.82]\u001b[A\n",
            "Batches:  47% 300/638 [00:06<00:07, 47.51it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.82]\u001b[A\n",
            "Batches:  48% 305/638 [00:06<00:07, 42.46it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:  49% 310/638 [00:06<00:07, 43.52it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:  49% 315/638 [00:07<00:07, 43.95it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:  50% 321/638 [00:07<00:07, 41.03it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  51% 327/638 [00:07<00:07, 44.01it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  52% 333/638 [00:07<00:06, 46.75it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  53% 339/638 [00:07<00:06, 48.21it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  54% 344/638 [00:07<00:06, 42.68it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  55% 350/638 [00:07<00:06, 45.26it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  56% 356/638 [00:07<00:05, 47.33it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  57% 361/638 [00:08<00:06, 43.07it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:  57% 366/638 [00:08<00:07, 37.17it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:  58% 372/638 [00:08<00:06, 40.72it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:  59% 378/638 [00:08<00:05, 43.53it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.81]\u001b[A\n",
            "Batches:  60% 383/638 [00:08<00:06, 40.30it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  61% 389/638 [00:08<00:05, 43.87it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  62% 395/638 [00:08<00:05, 46.94it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  63% 401/638 [00:09<00:05, 43.63it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  64% 407/638 [00:09<00:04, 46.44it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  65% 413/638 [00:09<00:04, 48.53it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  66% 419/638 [00:09<00:04, 49.42it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  67% 425/638 [00:09<00:04, 44.53it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.83]\u001b[A\n",
            "Batches:  68% 431/638 [00:09<00:04, 46.83it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.83]\u001b[A\n",
            "Batches:  68% 437/638 [00:09<00:04, 48.49it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.83]\u001b[A\n",
            "Batches:  69% 442/638 [00:09<00:04, 43.54it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  70% 448/638 [00:10<00:04, 46.29it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  71% 454/638 [00:10<00:03, 48.30it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  72% 460/638 [00:10<00:03, 49.83it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  73% 466/638 [00:10<00:04, 35.33it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  74% 471/638 [00:10<00:04, 38.60it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  75% 477/638 [00:10<00:03, 41.50it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  76% 482/638 [00:10<00:04, 38.74it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.82]\u001b[A\n",
            "Batches:  76% 488/638 [00:11<00:03, 42.22it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.82]\u001b[A\n",
            "Batches:  77% 494/638 [00:11<00:03, 44.64it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.82]\u001b[A\n",
            "Batches:  78% 500/638 [00:11<00:02, 46.36it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.82]\u001b[A\n",
            "Batches:  79% 505/638 [00:11<00:03, 42.04it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  80% 511/638 [00:11<00:02, 44.98it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  81% 517/638 [00:11<00:02, 46.94it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  82% 522/638 [00:11<00:02, 42.06it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  83% 528/638 [00:11<00:02, 44.65it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  84% 534/638 [00:11<00:02, 46.99it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  85% 540/638 [00:12<00:01, 49.05it/s, dev_acc=0.79, epoch=1, loss=0.14, train_acc=0.81]\u001b[A\n",
            "Batches:  86% 546/638 [00:12<00:02, 43.96it/s, dev_acc=0.80, epoch=1, loss=0.16, train_acc=0.82]\u001b[A\n",
            "Batches:  87% 552/638 [00:12<00:01, 45.80it/s, dev_acc=0.80, epoch=1, loss=0.16, train_acc=0.82]\u001b[A\n",
            "Batches:  87% 558/638 [00:12<00:01, 47.64it/s, dev_acc=0.80, epoch=1, loss=0.16, train_acc=0.82]\u001b[A\n",
            "Batches:  88% 563/638 [00:12<00:02, 35.35it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  89% 569/638 [00:12<00:01, 39.41it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  90% 575/638 [00:12<00:01, 42.81it/s, dev_acc=0.79, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  91% 581/638 [00:13<00:01, 41.09it/s, dev_acc=0.80, epoch=1, loss=0.15, train_acc=0.83]\u001b[A\n",
            "Batches:  92% 587/638 [00:13<00:01, 44.20it/s, dev_acc=0.80, epoch=1, loss=0.15, train_acc=0.83]\u001b[A\n",
            "Batches:  93% 593/638 [00:13<00:00, 46.34it/s, dev_acc=0.80, epoch=1, loss=0.15, train_acc=0.83]\u001b[A\n",
            "Batches:  94% 599/638 [00:13<00:00, 48.24it/s, dev_acc=0.80, epoch=1, loss=0.15, train_acc=0.83]\u001b[A\n",
            "Batches:  95% 605/638 [00:13<00:00, 44.00it/s, dev_acc=0.79, epoch=1, loss=0.16, train_acc=0.82]\u001b[A\n",
            "Batches:  96% 611/638 [00:13<00:00, 46.48it/s, dev_acc=0.79, epoch=1, loss=0.16, train_acc=0.82]\u001b[A\n",
            "Batches:  97% 617/638 [00:13<00:00, 48.23it/s, dev_acc=0.79, epoch=1, loss=0.16, train_acc=0.82]\u001b[A\n",
            "Batches:  97% 622/638 [00:13<00:00, 43.01it/s, dev_acc=0.80, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  98% 628/638 [00:14<00:00, 46.14it/s, dev_acc=0.80, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Batches:  99% 634/638 [00:14<00:00, 47.94it/s, dev_acc=0.80, epoch=1, loss=0.15, train_acc=0.82]\u001b[A\n",
            "Epochs:  20% 2/10 [00:36<02:34, 19.36s/it]\n",
            "Batches:   0% 0/638 [00:00<?, ?it/s, acc=0]\u001b[A\n",
            "Batches:   0% 2/638 [00:00<00:32, 19.49it/s, dev_acc=0.80, epoch=2, loss=0.14, train_acc=0.82]\u001b[A\n",
            "Batches:   1% 8/638 [00:00<00:26, 24.08it/s, dev_acc=0.80, epoch=2, loss=0.14, train_acc=0.82]\u001b[A\n",
            "Batches:   2% 14/638 [00:00<00:21, 28.83it/s, dev_acc=0.80, epoch=2, loss=0.14, train_acc=0.82]\u001b[A\n",
            "Batches:   3% 20/638 [00:00<00:18, 33.70it/s, dev_acc=0.80, epoch=2, loss=0.14, train_acc=0.82]\u001b[A\n",
            "Batches:   4% 24/638 [00:00<00:21, 27.92it/s, dev_acc=0.79, epoch=2, loss=0.12, train_acc=0.82]\u001b[A\n",
            "Batches:   5% 30/638 [00:00<00:18, 32.61it/s, dev_acc=0.79, epoch=2, loss=0.12, train_acc=0.82]\u001b[A\n",
            "Batches:   6% 36/638 [00:00<00:16, 36.77it/s, dev_acc=0.79, epoch=2, loss=0.12, train_acc=0.82]\u001b[A\n",
            "Batches:   6% 41/638 [00:01<00:17, 33.97it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:   7% 47/638 [00:01<00:15, 37.99it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:   8% 53/638 [00:01<00:14, 41.40it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:   9% 59/638 [00:01<00:13, 43.79it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  10% 64/638 [00:01<00:14, 40.39it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  11% 70/638 [00:01<00:13, 43.36it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  12% 76/638 [00:01<00:12, 46.01it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  13% 81/638 [00:01<00:13, 41.45it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  14% 87/638 [00:02<00:12, 44.80it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  15% 93/638 [00:02<00:11, 47.51it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  16% 99/638 [00:02<00:11, 48.86it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  16% 105/638 [00:02<00:12, 44.21it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.82]\u001b[A\n",
            "Batches:  17% 111/638 [00:02<00:11, 46.52it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.82]\u001b[A\n",
            "Batches:  18% 117/638 [00:02<00:10, 47.82it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.82]\u001b[A\n",
            "Batches:  19% 122/638 [00:02<00:12, 42.56it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  20% 127/638 [00:02<00:13, 37.06it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  21% 133/638 [00:03<00:12, 40.72it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  22% 139/638 [00:03<00:11, 43.98it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  23% 144/638 [00:03<00:12, 40.33it/s, dev_acc=0.79, epoch=2, loss=0.15, train_acc=0.83]\u001b[A\n",
            "Batches:  24% 150/638 [00:03<00:11, 43.10it/s, dev_acc=0.79, epoch=2, loss=0.15, train_acc=0.83]\u001b[A\n",
            "Batches:  24% 156/638 [00:03<00:10, 45.40it/s, dev_acc=0.79, epoch=2, loss=0.15, train_acc=0.83]\u001b[A\n",
            "Batches:  25% 161/638 [00:03<00:11, 40.87it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  26% 167/638 [00:03<00:10, 43.68it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  27% 173/638 [00:03<00:10, 45.74it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  28% 179/638 [00:04<00:09, 47.22it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  29% 184/638 [00:04<00:10, 42.50it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  30% 190/638 [00:04<00:09, 45.46it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  31% 196/638 [00:04<00:09, 47.77it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  32% 201/638 [00:04<00:10, 42.63it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  32% 207/638 [00:04<00:09, 45.40it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  33% 213/638 [00:04<00:08, 47.73it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  34% 219/638 [00:04<00:08, 49.45it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  35% 225/638 [00:05<00:10, 37.65it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  36% 231/638 [00:05<00:09, 41.01it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  37% 237/638 [00:05<00:09, 44.04it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  38% 242/638 [00:05<00:09, 40.43it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  39% 248/638 [00:05<00:09, 43.03it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  40% 254/638 [00:05<00:08, 45.48it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  41% 259/638 [00:05<00:08, 46.65it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  41% 264/638 [00:06<00:08, 41.72it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  42% 270/638 [00:06<00:08, 44.51it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  43% 276/638 [00:06<00:07, 47.11it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  44% 281/638 [00:06<00:08, 42.24it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.82]\u001b[A\n",
            "Batches:  45% 287/638 [00:06<00:07, 44.91it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.82]\u001b[A\n",
            "Batches:  46% 293/638 [00:06<00:07, 46.50it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.82]\u001b[A\n",
            "Batches:  47% 299/638 [00:06<00:07, 47.98it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.82]\u001b[A\n",
            "Batches:  48% 304/638 [00:06<00:07, 43.05it/s, dev_acc=0.79, epoch=2, loss=0.13, train_acc=0.83]\u001b[A\n",
            "Batches:  49% 310/638 [00:06<00:07, 45.91it/s, dev_acc=0.79, epoch=2, loss=0.13, train_acc=0.83]\u001b[A\n",
            "Batches:  50% 316/638 [00:07<00:06, 48.00it/s, dev_acc=0.79, epoch=2, loss=0.13, train_acc=0.83]\u001b[A\n",
            "Batches:  50% 321/638 [00:07<00:07, 42.24it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  51% 326/638 [00:07<00:08, 36.40it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  52% 332/638 [00:07<00:07, 40.02it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  53% 338/638 [00:07<00:06, 43.11it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  54% 343/638 [00:07<00:07, 40.32it/s, dev_acc=0.79, epoch=2, loss=0.12, train_acc=0.83]\u001b[A\n",
            "Batches:  55% 349/638 [00:07<00:06, 43.41it/s, dev_acc=0.79, epoch=2, loss=0.12, train_acc=0.83]\u001b[A\n",
            "Batches:  56% 355/638 [00:08<00:06, 46.39it/s, dev_acc=0.79, epoch=2, loss=0.12, train_acc=0.83]\u001b[A\n",
            "Batches:  57% 361/638 [00:08<00:06, 42.43it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  58% 367/638 [00:08<00:05, 45.25it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  58% 373/638 [00:08<00:05, 47.24it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  59% 379/638 [00:08<00:05, 48.46it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  60% 384/638 [00:08<00:05, 43.52it/s, dev_acc=0.79, epoch=2, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  61% 390/638 [00:08<00:05, 46.32it/s, dev_acc=0.79, epoch=2, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  62% 396/638 [00:08<00:04, 48.57it/s, dev_acc=0.79, epoch=2, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  63% 402/638 [00:09<00:05, 44.34it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  64% 408/638 [00:09<00:04, 46.13it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  65% 414/638 [00:09<00:04, 47.54it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  66% 420/638 [00:09<00:04, 49.15it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  67% 426/638 [00:09<00:06, 32.81it/s, dev_acc=0.79, epoch=2, loss=0.13, train_acc=0.83]\u001b[A\n",
            "Batches:  68% 432/638 [00:09<00:05, 36.76it/s, dev_acc=0.79, epoch=2, loss=0.13, train_acc=0.83]\u001b[A\n",
            "Batches:  69% 438/638 [00:09<00:04, 40.28it/s, dev_acc=0.79, epoch=2, loss=0.13, train_acc=0.83]\u001b[A\n",
            "Batches:  69% 443/638 [00:10<00:05, 38.26it/s, dev_acc=0.79, epoch=2, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  70% 449/638 [00:10<00:04, 41.72it/s, dev_acc=0.79, epoch=2, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  71% 455/638 [00:10<00:04, 44.47it/s, dev_acc=0.79, epoch=2, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  72% 461/638 [00:10<00:04, 41.19it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Batches:  73% 467/638 [00:10<00:03, 43.96it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Batches:  74% 473/638 [00:10<00:03, 46.06it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Batches:  75% 479/638 [00:10<00:03, 47.39it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Batches:  76% 484/638 [00:11<00:03, 41.93it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Batches:  77% 490/638 [00:11<00:03, 44.89it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Batches:  78% 496/638 [00:11<00:02, 47.70it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Batches:  79% 501/638 [00:11<00:03, 43.02it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Batches:  79% 507/638 [00:11<00:02, 46.05it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Batches:  80% 513/638 [00:11<00:02, 47.93it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Batches:  81% 519/638 [00:11<00:02, 49.41it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Batches:  82% 525/638 [00:11<00:02, 38.03it/s, dev_acc=0.79, epoch=2, loss=0.15, train_acc=0.84]\u001b[A\n",
            "Batches:  83% 531/638 [00:12<00:02, 42.00it/s, dev_acc=0.79, epoch=2, loss=0.15, train_acc=0.84]\u001b[A\n",
            "Batches:  84% 537/638 [00:12<00:02, 44.94it/s, dev_acc=0.79, epoch=2, loss=0.15, train_acc=0.84]\u001b[A\n",
            "Batches:  85% 542/638 [00:12<00:02, 41.02it/s, dev_acc=0.79, epoch=2, loss=0.13, train_acc=0.83]\u001b[A\n",
            "Batches:  86% 548/638 [00:12<00:02, 43.90it/s, dev_acc=0.79, epoch=2, loss=0.13, train_acc=0.83]\u001b[A\n",
            "Batches:  87% 554/638 [00:12<00:01, 46.17it/s, dev_acc=0.79, epoch=2, loss=0.13, train_acc=0.83]\u001b[A\n",
            "Batches:  88% 560/638 [00:12<00:01, 48.24it/s, dev_acc=0.79, epoch=2, loss=0.13, train_acc=0.83]\u001b[A\n",
            "Batches:  89% 566/638 [00:12<00:01, 44.01it/s, dev_acc=0.79, epoch=2, loss=0.13, train_acc=0.83]\u001b[A\n",
            "Batches:  90% 572/638 [00:12<00:01, 46.55it/s, dev_acc=0.79, epoch=2, loss=0.13, train_acc=0.83]\u001b[A\n",
            "Batches:  91% 578/638 [00:13<00:01, 48.55it/s, dev_acc=0.79, epoch=2, loss=0.13, train_acc=0.83]\u001b[A\n",
            "Batches:  92% 584/638 [00:13<00:01, 44.11it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Batches:  92% 590/638 [00:13<00:01, 46.33it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Batches:  93% 596/638 [00:13<00:00, 48.17it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Batches:  94% 601/638 [00:13<00:00, 42.59it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  95% 607/638 [00:13<00:00, 45.43it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  96% 613/638 [00:13<00:00, 47.88it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  97% 619/638 [00:13<00:00, 49.17it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.83]\u001b[A\n",
            "Batches:  98% 625/638 [00:14<00:00, 37.96it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Batches:  99% 631/638 [00:14<00:00, 41.54it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Batches: 100% 637/638 [00:14<00:00, 44.32it/s, dev_acc=0.79, epoch=2, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Epochs:  30% 3/10 [00:51<02:06, 18.01s/it]\n",
            "Batches:   0% 0/638 [00:00<?, ?it/s, acc=0]\u001b[A\n",
            "Batches:   0% 3/638 [00:00<00:23, 27.39it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:   1% 9/638 [00:00<00:19, 32.18it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:   2% 15/638 [00:00<00:17, 36.47it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:   3% 20/638 [00:00<00:16, 37.81it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:   4% 24/638 [00:00<00:17, 35.25it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:   5% 30/638 [00:00<00:15, 39.07it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:   6% 36/638 [00:00<00:14, 42.44it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:   6% 41/638 [00:00<00:15, 39.17it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:   7% 46/638 [00:01<00:14, 41.28it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:   8% 52/638 [00:01<00:13, 44.19it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:   9% 58/638 [00:01<00:12, 46.14it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  10% 63/638 [00:01<00:13, 42.13it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  11% 69/638 [00:01<00:12, 45.30it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  12% 75/638 [00:01<00:11, 46.99it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  13% 80/638 [00:01<00:11, 47.66it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  13% 85/638 [00:01<00:13, 41.94it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  14% 90/638 [00:02<00:15, 36.45it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  15% 95/638 [00:02<00:13, 39.46it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  16% 100/638 [00:02<00:12, 41.90it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  16% 105/638 [00:02<00:13, 38.80it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  17% 110/638 [00:02<00:12, 41.02it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  18% 115/638 [00:02<00:12, 43.09it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  19% 120/638 [00:02<00:11, 44.85it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  20% 125/638 [00:02<00:12, 40.52it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:  20% 130/638 [00:03<00:11, 42.85it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:  21% 135/638 [00:03<00:11, 44.34it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:  22% 141/638 [00:03<00:12, 41.18it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  23% 146/638 [00:03<00:11, 43.26it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  24% 151/638 [00:03<00:10, 44.96it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  24% 156/638 [00:03<00:10, 46.22it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  25% 161/638 [00:03<00:11, 40.88it/s, dev_acc=0.79, epoch=3, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Batches:  26% 167/638 [00:03<00:10, 44.20it/s, dev_acc=0.79, epoch=3, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Batches:  27% 173/638 [00:03<00:09, 46.52it/s, dev_acc=0.79, epoch=3, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Batches:  28% 179/638 [00:04<00:09, 47.92it/s, dev_acc=0.79, epoch=3, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Batches:  29% 184/638 [00:04<00:10, 42.18it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  30% 189/638 [00:04<00:12, 36.96it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  30% 194/638 [00:04<00:11, 39.37it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  31% 200/638 [00:04<00:10, 42.40it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  32% 205/638 [00:04<00:11, 39.19it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  33% 210/638 [00:04<00:10, 41.56it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  34% 216/638 [00:05<00:09, 43.77it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  35% 221/638 [00:05<00:10, 39.80it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:  36% 227/638 [00:05<00:09, 42.48it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:  36% 232/638 [00:05<00:09, 44.00it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:  37% 238/638 [00:05<00:08, 46.04it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:  38% 243/638 [00:05<00:09, 40.63it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  39% 249/638 [00:05<00:08, 43.89it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  40% 255/638 [00:05<00:08, 45.91it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  41% 260/638 [00:05<00:08, 45.84it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  42% 265/638 [00:06<00:09, 41.17it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:  42% 270/638 [00:06<00:08, 42.55it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:  43% 275/638 [00:06<00:08, 44.01it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:  44% 281/638 [00:06<00:08, 40.66it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  45% 286/638 [00:06<00:09, 36.34it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  46% 291/638 [00:06<00:08, 39.55it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  46% 296/638 [00:06<00:08, 41.69it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  47% 301/638 [00:07<00:08, 37.82it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  48% 306/638 [00:07<00:08, 39.52it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  49% 312/638 [00:07<00:07, 42.51it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  50% 318/638 [00:07<00:07, 44.91it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  51% 323/638 [00:07<00:07, 40.05it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:  51% 328/638 [00:07<00:07, 41.51it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:  52% 333/638 [00:07<00:07, 43.25it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:  53% 338/638 [00:07<00:06, 44.83it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:  54% 343/638 [00:08<00:07, 39.84it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  55% 348/638 [00:08<00:06, 41.97it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  55% 353/638 [00:08<00:06, 43.83it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  56% 359/638 [00:08<00:06, 46.07it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  57% 364/638 [00:08<00:06, 41.48it/s, dev_acc=0.80, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  58% 369/638 [00:08<00:06, 43.68it/s, dev_acc=0.80, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  59% 375/638 [00:08<00:05, 45.40it/s, dev_acc=0.80, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  60% 381/638 [00:08<00:06, 41.49it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  61% 386/638 [00:09<00:07, 35.46it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  61% 391/638 [00:09<00:06, 38.48it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  62% 397/638 [00:09<00:05, 41.32it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  63% 402/638 [00:09<00:06, 38.25it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  64% 408/638 [00:09<00:05, 41.63it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  65% 413/638 [00:09<00:05, 43.28it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  66% 418/638 [00:09<00:04, 44.87it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  66% 423/638 [00:09<00:05, 39.84it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  67% 428/638 [00:10<00:04, 42.29it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  68% 433/638 [00:10<00:04, 44.11it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  69% 438/638 [00:10<00:04, 45.28it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  69% 443/638 [00:10<00:04, 40.41it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  70% 449/638 [00:10<00:04, 42.78it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  71% 455/638 [00:10<00:04, 44.68it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  72% 460/638 [00:10<00:03, 45.69it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  73% 465/638 [00:10<00:04, 40.28it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  74% 471/638 [00:11<00:03, 43.31it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  75% 476/638 [00:11<00:03, 44.71it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  75% 481/638 [00:11<00:03, 39.57it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  76% 486/638 [00:11<00:04, 35.48it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  77% 492/638 [00:11<00:03, 39.32it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  78% 497/638 [00:11<00:03, 41.82it/s, dev_acc=0.79, epoch=3, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  79% 502/638 [00:11<00:03, 38.01it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  79% 507/638 [00:11<00:03, 40.90it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  80% 512/638 [00:12<00:02, 42.33it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  81% 517/638 [00:12<00:02, 42.88it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  82% 522/638 [00:12<00:03, 38.26it/s, dev_acc=0.80, epoch=3, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  83% 527/638 [00:12<00:02, 40.94it/s, dev_acc=0.80, epoch=3, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  84% 533/638 [00:12<00:02, 43.90it/s, dev_acc=0.80, epoch=3, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  84% 539/638 [00:12<00:02, 45.61it/s, dev_acc=0.80, epoch=3, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  85% 544/638 [00:12<00:02, 40.57it/s, dev_acc=0.79, epoch=3, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  86% 550/638 [00:12<00:02, 43.52it/s, dev_acc=0.79, epoch=3, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  87% 556/638 [00:13<00:01, 45.31it/s, dev_acc=0.79, epoch=3, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  88% 561/638 [00:13<00:01, 40.30it/s, dev_acc=0.79, epoch=3, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  89% 566/638 [00:13<00:01, 42.47it/s, dev_acc=0.79, epoch=3, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  89% 571/638 [00:13<00:01, 44.30it/s, dev_acc=0.79, epoch=3, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  90% 576/638 [00:13<00:01, 45.53it/s, dev_acc=0.79, epoch=3, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  91% 581/638 [00:13<00:01, 40.53it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  92% 586/638 [00:13<00:01, 34.70it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  93% 591/638 [00:13<00:01, 37.69it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  93% 596/638 [00:14<00:01, 40.06it/s, dev_acc=0.79, epoch=3, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  94% 601/638 [00:14<00:00, 37.64it/s, dev_acc=0.79, epoch=3, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Batches:  95% 606/638 [00:14<00:00, 40.46it/s, dev_acc=0.79, epoch=3, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Batches:  96% 611/638 [00:14<00:00, 42.46it/s, dev_acc=0.79, epoch=3, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Batches:  97% 617/638 [00:14<00:00, 44.44it/s, dev_acc=0.79, epoch=3, loss=0.14, train_acc=0.84]\u001b[A\n",
            "Batches:  97% 622/638 [00:14<00:00, 40.58it/s, dev_acc=0.79, epoch=3, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  98% 627/638 [00:14<00:00, 42.81it/s, dev_acc=0.79, epoch=3, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  99% 632/638 [00:14<00:00, 44.68it/s, dev_acc=0.79, epoch=3, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches: 100% 637/638 [00:15<00:00, 45.99it/s, dev_acc=0.79, epoch=3, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Epochs:  40% 4/10 [01:06<01:43, 17.26s/it]\n",
            "Batches:   0% 0/638 [00:00<?, ?it/s, acc=0]\u001b[A\n",
            "Batches:   0% 3/638 [00:00<00:24, 25.62it/s, dev_acc=0.79, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:   1% 9/638 [00:00<00:20, 30.38it/s, dev_acc=0.79, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:   2% 15/638 [00:00<00:17, 34.86it/s, dev_acc=0.79, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:   3% 21/638 [00:00<00:17, 35.42it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:   4% 27/638 [00:00<00:15, 39.32it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:   5% 33/638 [00:00<00:14, 42.81it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:   6% 39/638 [00:00<00:13, 45.26it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:   7% 44/638 [00:00<00:14, 41.07it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:   8% 49/638 [00:01<00:16, 35.32it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:   9% 55/638 [00:01<00:14, 38.87it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.84]\u001b[A\n",
            "Batches:  10% 61/638 [00:01<00:15, 38.10it/s, dev_acc=0.79, epoch=4, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:  11% 67/638 [00:01<00:13, 41.19it/s, dev_acc=0.79, epoch=4, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:  11% 73/638 [00:01<00:12, 43.94it/s, dev_acc=0.79, epoch=4, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:  12% 79/638 [00:01<00:12, 45.97it/s, dev_acc=0.79, epoch=4, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:  13% 84/638 [00:01<00:13, 41.90it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  14% 90/638 [00:02<00:12, 44.45it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  15% 96/638 [00:02<00:11, 46.64it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  16% 101/638 [00:02<00:12, 42.24it/s, dev_acc=0.79, epoch=4, loss=0.11, train_acc=0.85]\u001b[A\n",
            "Batches:  17% 107/638 [00:02<00:11, 44.51it/s, dev_acc=0.79, epoch=4, loss=0.11, train_acc=0.85]\u001b[A\n",
            "Batches:  18% 113/638 [00:02<00:11, 46.39it/s, dev_acc=0.79, epoch=4, loss=0.11, train_acc=0.85]\u001b[A\n",
            "Batches:  18% 118/638 [00:02<00:11, 47.12it/s, dev_acc=0.79, epoch=4, loss=0.11, train_acc=0.85]\u001b[A\n",
            "Batches:  19% 123/638 [00:02<00:12, 42.04it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  20% 129/638 [00:02<00:11, 44.42it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  21% 135/638 [00:03<00:10, 46.76it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  22% 141/638 [00:03<00:11, 43.16it/s, dev_acc=0.78, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  23% 147/638 [00:03<00:10, 45.60it/s, dev_acc=0.78, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  24% 152/638 [00:03<00:12, 38.79it/s, dev_acc=0.78, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  25% 158/638 [00:03<00:11, 41.85it/s, dev_acc=0.78, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  26% 163/638 [00:03<00:12, 39.25it/s, dev_acc=0.78, epoch=4, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:  26% 168/638 [00:03<00:11, 41.70it/s, dev_acc=0.78, epoch=4, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:  27% 174/638 [00:03<00:10, 44.21it/s, dev_acc=0.78, epoch=4, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:  28% 180/638 [00:04<00:09, 46.24it/s, dev_acc=0.78, epoch=4, loss=0.12, train_acc=0.84]\u001b[A\n",
            "Batches:  29% 185/638 [00:04<00:10, 41.67it/s, dev_acc=0.79, epoch=4, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  30% 191/638 [00:04<00:10, 44.35it/s, dev_acc=0.79, epoch=4, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  31% 197/638 [00:04<00:09, 46.48it/s, dev_acc=0.79, epoch=4, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  32% 202/638 [00:04<00:10, 42.16it/s, dev_acc=0.79, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  33% 208/638 [00:04<00:09, 44.94it/s, dev_acc=0.79, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  34% 214/638 [00:04<00:08, 47.13it/s, dev_acc=0.79, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  34% 219/638 [00:04<00:08, 47.64it/s, dev_acc=0.79, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  35% 224/638 [00:05<00:09, 42.03it/s, dev_acc=0.78, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  36% 230/638 [00:05<00:09, 44.56it/s, dev_acc=0.78, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  37% 236/638 [00:05<00:08, 46.79it/s, dev_acc=0.78, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  38% 241/638 [00:05<00:09, 42.25it/s, dev_acc=0.78, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  39% 247/638 [00:05<00:08, 45.42it/s, dev_acc=0.78, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  39% 252/638 [00:05<00:10, 38.01it/s, dev_acc=0.78, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  40% 258/638 [00:05<00:09, 41.24it/s, dev_acc=0.78, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  41% 263/638 [00:06<00:09, 38.58it/s, dev_acc=0.79, epoch=4, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  42% 269/638 [00:06<00:08, 41.62it/s, dev_acc=0.79, epoch=4, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  43% 275/638 [00:06<00:08, 44.14it/s, dev_acc=0.79, epoch=4, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  44% 281/638 [00:06<00:08, 41.28it/s, dev_acc=0.79, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  45% 286/638 [00:06<00:08, 43.54it/s, dev_acc=0.79, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  46% 292/638 [00:06<00:07, 45.75it/s, dev_acc=0.79, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  47% 298/638 [00:06<00:07, 47.23it/s, dev_acc=0.79, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  47% 303/638 [00:06<00:07, 42.12it/s, dev_acc=0.78, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  48% 309/638 [00:07<00:07, 44.66it/s, dev_acc=0.78, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  49% 315/638 [00:07<00:06, 46.68it/s, dev_acc=0.78, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  50% 321/638 [00:07<00:07, 43.44it/s, dev_acc=0.78, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  51% 327/638 [00:07<00:06, 46.23it/s, dev_acc=0.78, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  52% 333/638 [00:07<00:06, 47.55it/s, dev_acc=0.78, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  53% 339/638 [00:07<00:06, 48.93it/s, dev_acc=0.78, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  54% 344/638 [00:07<00:06, 43.60it/s, dev_acc=0.78, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  55% 349/638 [00:07<00:07, 37.23it/s, dev_acc=0.78, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  56% 355/638 [00:08<00:06, 40.79it/s, dev_acc=0.78, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  57% 361/638 [00:08<00:07, 39.09it/s, dev_acc=0.78, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  58% 367/638 [00:08<00:06, 42.72it/s, dev_acc=0.78, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  58% 373/638 [00:08<00:05, 45.59it/s, dev_acc=0.78, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  59% 379/638 [00:08<00:05, 47.57it/s, dev_acc=0.78, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  60% 384/638 [00:08<00:05, 42.67it/s, dev_acc=0.78, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  61% 389/638 [00:08<00:05, 44.49it/s, dev_acc=0.78, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  62% 395/638 [00:08<00:05, 46.52it/s, dev_acc=0.78, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  63% 401/638 [00:09<00:05, 42.63it/s, dev_acc=0.79, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  64% 407/638 [00:09<00:05, 45.22it/s, dev_acc=0.79, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  65% 413/638 [00:09<00:04, 46.89it/s, dev_acc=0.79, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  66% 419/638 [00:09<00:04, 48.48it/s, dev_acc=0.79, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  66% 424/638 [00:09<00:04, 43.13it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  67% 430/638 [00:09<00:04, 45.44it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  68% 436/638 [00:09<00:04, 47.19it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  69% 441/638 [00:09<00:04, 41.64it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  70% 447/638 [00:10<00:04, 44.07it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  71% 452/638 [00:10<00:04, 37.81it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  72% 458/638 [00:10<00:04, 41.00it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  73% 463/638 [00:10<00:04, 39.04it/s, dev_acc=0.79, epoch=4, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  74% 469/638 [00:10<00:03, 42.48it/s, dev_acc=0.79, epoch=4, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  74% 475/638 [00:10<00:03, 44.94it/s, dev_acc=0.79, epoch=4, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  75% 481/638 [00:10<00:03, 41.89it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  76% 487/638 [00:11<00:03, 44.64it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  77% 493/638 [00:11<00:03, 46.31it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  78% 499/638 [00:11<00:02, 48.59it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  79% 504/638 [00:11<00:03, 43.64it/s, dev_acc=0.78, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  80% 510/638 [00:11<00:02, 46.31it/s, dev_acc=0.78, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  81% 516/638 [00:11<00:02, 48.20it/s, dev_acc=0.78, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  82% 521/638 [00:11<00:02, 42.98it/s, dev_acc=0.78, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  83% 527/638 [00:11<00:02, 45.70it/s, dev_acc=0.78, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  84% 533/638 [00:12<00:02, 47.75it/s, dev_acc=0.78, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  84% 539/638 [00:12<00:02, 49.37it/s, dev_acc=0.78, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  85% 545/638 [00:12<00:02, 44.94it/s, dev_acc=0.78, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  86% 550/638 [00:12<00:02, 38.33it/s, dev_acc=0.78, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  87% 556/638 [00:12<00:01, 41.62it/s, dev_acc=0.78, epoch=4, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  88% 561/638 [00:12<00:01, 38.95it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  89% 567/638 [00:12<00:01, 42.63it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  90% 573/638 [00:12<00:01, 45.63it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  91% 579/638 [00:13<00:01, 48.00it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  92% 585/638 [00:13<00:01, 43.56it/s, dev_acc=0.78, epoch=4, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  93% 591/638 [00:13<00:01, 46.66it/s, dev_acc=0.78, epoch=4, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  94% 597/638 [00:13<00:00, 48.77it/s, dev_acc=0.78, epoch=4, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  95% 603/638 [00:13<00:00, 44.25it/s, dev_acc=0.79, epoch=4, loss=0.15, train_acc=0.85]\u001b[A\n",
            "Batches:  95% 609/638 [00:13<00:00, 47.15it/s, dev_acc=0.79, epoch=4, loss=0.15, train_acc=0.85]\u001b[A\n",
            "Batches:  96% 615/638 [00:13<00:00, 49.15it/s, dev_acc=0.79, epoch=4, loss=0.15, train_acc=0.85]\u001b[A\n",
            "Batches:  97% 621/638 [00:13<00:00, 45.25it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  98% 627/638 [00:14<00:00, 47.81it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  99% 633/638 [00:14<00:00, 49.71it/s, dev_acc=0.79, epoch=4, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Epochs:  50% 5/10 [01:21<01:22, 16.50s/it]\n",
            "Batches:   0% 0/638 [00:00<?, ?it/s, acc=0]\u001b[A\n",
            "Batches:   0% 3/638 [00:00<00:23, 27.53it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:   1% 9/638 [00:00<00:19, 32.53it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:   2% 12/638 [00:00<00:26, 23.60it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:   3% 18/638 [00:00<00:21, 28.38it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:   3% 22/638 [00:00<00:20, 29.51it/s, dev_acc=0.79, epoch=5, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:   4% 28/638 [00:00<00:17, 34.28it/s, dev_acc=0.79, epoch=5, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:   5% 34/638 [00:00<00:15, 38.97it/s, dev_acc=0.79, epoch=5, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:   6% 40/638 [00:00<00:13, 43.17it/s, dev_acc=0.79, epoch=5, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:   7% 45/638 [00:01<00:14, 40.60it/s, dev_acc=0.79, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:   8% 51/638 [00:01<00:13, 43.53it/s, dev_acc=0.79, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:   9% 57/638 [00:01<00:12, 46.62it/s, dev_acc=0.79, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  10% 62/638 [00:01<00:13, 42.60it/s, dev_acc=0.79, epoch=5, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  11% 68/638 [00:01<00:12, 45.29it/s, dev_acc=0.79, epoch=5, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  12% 74/638 [00:01<00:11, 47.29it/s, dev_acc=0.79, epoch=5, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  13% 80/638 [00:01<00:11, 48.78it/s, dev_acc=0.79, epoch=5, loss=0.14, train_acc=0.85]\u001b[A\n",
            "Batches:  13% 86/638 [00:01<00:12, 44.06it/s, dev_acc=0.79, epoch=5, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  14% 92/638 [00:02<00:11, 46.31it/s, dev_acc=0.79, epoch=5, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  15% 98/638 [00:02<00:11, 48.10it/s, dev_acc=0.79, epoch=5, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  16% 103/638 [00:02<00:12, 43.38it/s, dev_acc=0.78, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  17% 109/638 [00:02<00:11, 45.69it/s, dev_acc=0.78, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  18% 114/638 [00:02<00:14, 35.56it/s, dev_acc=0.78, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  19% 119/638 [00:02<00:13, 38.44it/s, dev_acc=0.78, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  19% 124/638 [00:02<00:14, 36.63it/s, dev_acc=0.79, epoch=5, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  20% 130/638 [00:03<00:12, 39.95it/s, dev_acc=0.79, epoch=5, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  21% 136/638 [00:03<00:11, 42.96it/s, dev_acc=0.79, epoch=5, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  22% 141/638 [00:03<00:12, 39.39it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  23% 147/638 [00:03<00:11, 42.42it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  24% 152/638 [00:03<00:10, 44.35it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  25% 158/638 [00:03<00:10, 46.62it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  26% 163/638 [00:03<00:11, 42.34it/s, dev_acc=0.78, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  26% 169/638 [00:03<00:10, 44.83it/s, dev_acc=0.78, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  27% 175/638 [00:04<00:09, 46.69it/s, dev_acc=0.78, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  28% 181/638 [00:04<00:10, 42.99it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  29% 187/638 [00:04<00:09, 45.46it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  30% 193/638 [00:04<00:09, 47.12it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  31% 199/638 [00:04<00:09, 48.40it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  32% 204/638 [00:04<00:10, 42.87it/s, dev_acc=0.78, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  33% 210/638 [00:04<00:11, 37.96it/s, dev_acc=0.78, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  34% 216/638 [00:05<00:10, 41.80it/s, dev_acc=0.78, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  35% 221/638 [00:05<00:10, 39.39it/s, dev_acc=0.78, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  36% 227/638 [00:05<00:09, 42.36it/s, dev_acc=0.78, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  37% 233/638 [00:05<00:09, 44.79it/s, dev_acc=0.78, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  37% 239/638 [00:05<00:08, 47.21it/s, dev_acc=0.78, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  38% 244/638 [00:05<00:09, 42.78it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  39% 250/638 [00:05<00:08, 45.42it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  40% 256/638 [00:05<00:08, 47.60it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  41% 261/638 [00:06<00:08, 42.14it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  42% 267/638 [00:06<00:08, 44.65it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  43% 273/638 [00:06<00:07, 46.45it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  44% 278/638 [00:06<00:07, 47.34it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  44% 283/638 [00:06<00:08, 42.44it/s, dev_acc=0.79, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  45% 289/638 [00:06<00:07, 44.72it/s, dev_acc=0.79, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  46% 295/638 [00:06<00:07, 46.18it/s, dev_acc=0.79, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  47% 301/638 [00:06<00:07, 42.27it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  48% 307/638 [00:07<00:07, 44.64it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  49% 312/638 [00:07<00:08, 37.88it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  50% 318/638 [00:07<00:07, 41.04it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  51% 323/638 [00:07<00:08, 38.63it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  52% 329/638 [00:07<00:07, 41.79it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  52% 334/638 [00:07<00:06, 43.91it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  53% 340/638 [00:07<00:06, 45.96it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  54% 345/638 [00:07<00:06, 41.89it/s, dev_acc=0.79, epoch=5, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  55% 351/638 [00:08<00:06, 45.07it/s, dev_acc=0.79, epoch=5, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  56% 356/638 [00:08<00:06, 44.92it/s, dev_acc=0.79, epoch=5, loss=0.12, train_acc=0.85]\u001b[A\n",
            "Batches:  57% 361/638 [00:08<00:06, 41.67it/s, dev_acc=0.79, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  58% 367/638 [00:08<00:06, 44.76it/s, dev_acc=0.79, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  58% 373/638 [00:08<00:05, 47.23it/s, dev_acc=0.79, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  59% 379/638 [00:08<00:05, 49.08it/s, dev_acc=0.79, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  60% 385/638 [00:08<00:05, 44.86it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  61% 391/638 [00:08<00:05, 47.33it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  62% 397/638 [00:09<00:04, 48.63it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  63% 402/638 [00:09<00:05, 42.95it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  64% 408/638 [00:09<00:05, 45.26it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  65% 413/638 [00:09<00:05, 38.81it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  66% 419/638 [00:09<00:05, 42.20it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  66% 424/638 [00:09<00:05, 39.24it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  67% 430/638 [00:09<00:04, 42.98it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  68% 436/638 [00:09<00:04, 46.28it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  69% 441/638 [00:10<00:04, 42.46it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  70% 447/638 [00:10<00:04, 45.54it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  71% 453/638 [00:10<00:03, 48.09it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  72% 459/638 [00:10<00:03, 50.16it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  73% 465/638 [00:10<00:03, 45.25it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  74% 471/638 [00:10<00:03, 47.23it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  75% 477/638 [00:10<00:03, 48.51it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  76% 482/638 [00:10<00:03, 43.02it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  76% 488/638 [00:11<00:03, 46.12it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  77% 494/638 [00:11<00:02, 48.40it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  78% 500/638 [00:11<00:02, 49.33it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  79% 506/638 [00:11<00:02, 44.27it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  80% 511/638 [00:11<00:03, 38.45it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  81% 517/638 [00:11<00:02, 41.89it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  82% 522/638 [00:11<00:02, 39.50it/s, dev_acc=0.79, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  83% 528/638 [00:11<00:02, 43.16it/s, dev_acc=0.79, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  84% 534/638 [00:12<00:02, 46.11it/s, dev_acc=0.79, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  85% 540/638 [00:12<00:02, 47.87it/s, dev_acc=0.79, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  85% 545/638 [00:12<00:02, 42.89it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  86% 551/638 [00:12<00:01, 45.48it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  87% 557/638 [00:12<00:01, 47.62it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  88% 562/638 [00:12<00:01, 42.36it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  89% 568/638 [00:12<00:01, 44.80it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  90% 574/638 [00:12<00:01, 46.93it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  91% 580/638 [00:13<00:01, 48.19it/s, dev_acc=0.79, epoch=5, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  92% 585/638 [00:13<00:01, 40.33it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  93% 591/638 [00:13<00:01, 44.00it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  94% 597/638 [00:13<00:00, 46.95it/s, dev_acc=0.78, epoch=5, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  94% 602/638 [00:13<00:00, 42.53it/s, dev_acc=0.78, epoch=5, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  95% 608/638 [00:13<00:00, 45.38it/s, dev_acc=0.78, epoch=5, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  96% 613/638 [00:13<00:00, 38.21it/s, dev_acc=0.78, epoch=5, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  97% 619/638 [00:13<00:00, 41.92it/s, dev_acc=0.78, epoch=5, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  98% 624/638 [00:14<00:00, 39.57it/s, dev_acc=0.78, epoch=5, loss=0.14, train_acc=0.86]\u001b[A\n",
            "Batches:  99% 630/638 [00:14<00:00, 42.77it/s, dev_acc=0.78, epoch=5, loss=0.14, train_acc=0.86]\u001b[A\n",
            "Batches: 100% 636/638 [00:14<00:00, 45.86it/s, dev_acc=0.78, epoch=5, loss=0.14, train_acc=0.86]\u001b[A\n",
            "Epochs:  60% 6/10 [01:36<01:03, 16.00s/it]\n",
            "Batches:   0% 0/638 [00:00<?, ?it/s, acc=0]\u001b[A\n",
            "Batches:   0% 3/638 [00:00<00:24, 25.70it/s, dev_acc=0.79, epoch=6, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:   1% 9/638 [00:00<00:20, 30.38it/s, dev_acc=0.79, epoch=6, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:   2% 15/638 [00:00<00:17, 35.18it/s, dev_acc=0.79, epoch=6, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:   3% 21/638 [00:00<00:17, 35.87it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:   4% 27/638 [00:00<00:15, 39.65it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:   5% 33/638 [00:00<00:14, 43.19it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:   6% 39/638 [00:00<00:13, 45.73it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:   7% 44/638 [00:00<00:14, 41.66it/s, dev_acc=0.78, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:   8% 50/638 [00:01<00:13, 44.39it/s, dev_acc=0.78, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:   9% 55/638 [00:01<00:12, 45.78it/s, dev_acc=0.78, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  10% 61/638 [00:01<00:13, 42.27it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  11% 67/638 [00:01<00:12, 44.64it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  11% 72/638 [00:01<00:14, 37.93it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  12% 78/638 [00:01<00:13, 41.31it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  13% 83/638 [00:01<00:14, 38.98it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  14% 88/638 [00:02<00:13, 41.39it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  15% 94/638 [00:02<00:12, 44.16it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  16% 100/638 [00:02<00:11, 46.36it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  16% 105/638 [00:02<00:12, 42.08it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  17% 110/638 [00:02<00:11, 44.16it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  18% 115/638 [00:02<00:11, 45.75it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  19% 121/638 [00:02<00:12, 42.15it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  20% 127/638 [00:02<00:11, 44.61it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  21% 133/638 [00:02<00:10, 46.79it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  22% 138/638 [00:03<00:10, 47.54it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  22% 143/638 [00:03<00:11, 42.60it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  23% 149/638 [00:03<00:10, 45.57it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  24% 155/638 [00:03<00:10, 47.35it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  25% 161/638 [00:03<00:11, 43.32it/s, dev_acc=0.78, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  26% 167/638 [00:03<00:10, 45.66it/s, dev_acc=0.78, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  27% 172/638 [00:03<00:12, 38.67it/s, dev_acc=0.78, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  28% 178/638 [00:04<00:10, 42.43it/s, dev_acc=0.78, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  29% 183/638 [00:04<00:11, 39.04it/s, dev_acc=0.79, epoch=6, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  30% 189/638 [00:04<00:10, 42.00it/s, dev_acc=0.79, epoch=6, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  31% 195/638 [00:04<00:09, 44.54it/s, dev_acc=0.79, epoch=6, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  32% 201/638 [00:04<00:10, 41.86it/s, dev_acc=0.79, epoch=6, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  32% 207/638 [00:04<00:09, 44.74it/s, dev_acc=0.79, epoch=6, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  33% 213/638 [00:04<00:09, 46.91it/s, dev_acc=0.79, epoch=6, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  34% 219/638 [00:04<00:08, 47.92it/s, dev_acc=0.79, epoch=6, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  35% 224/638 [00:05<00:09, 41.54it/s, dev_acc=0.78, epoch=6, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  36% 230/638 [00:05<00:09, 44.67it/s, dev_acc=0.78, epoch=6, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  37% 236/638 [00:05<00:08, 46.57it/s, dev_acc=0.78, epoch=6, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  38% 241/638 [00:05<00:09, 42.35it/s, dev_acc=0.79, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  39% 247/638 [00:05<00:08, 45.07it/s, dev_acc=0.79, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  40% 253/638 [00:05<00:08, 47.11it/s, dev_acc=0.79, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  41% 259/638 [00:05<00:07, 48.57it/s, dev_acc=0.79, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  41% 264/638 [00:05<00:08, 43.02it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  42% 270/638 [00:06<00:08, 45.61it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  43% 275/638 [00:06<00:09, 38.86it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  44% 281/638 [00:06<00:09, 38.14it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  45% 287/638 [00:06<00:08, 41.48it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  46% 293/638 [00:06<00:07, 44.27it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  47% 299/638 [00:06<00:07, 46.14it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  48% 304/638 [00:06<00:08, 41.73it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  48% 309/638 [00:06<00:07, 43.88it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  49% 315/638 [00:07<00:07, 46.13it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  50% 321/638 [00:07<00:07, 42.88it/s, dev_acc=0.79, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  51% 327/638 [00:07<00:06, 45.30it/s, dev_acc=0.79, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  52% 333/638 [00:07<00:06, 47.51it/s, dev_acc=0.79, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  53% 339/638 [00:07<00:06, 49.53it/s, dev_acc=0.79, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  54% 345/638 [00:07<00:06, 44.72it/s, dev_acc=0.78, epoch=6, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  55% 351/638 [00:07<00:06, 47.05it/s, dev_acc=0.78, epoch=6, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  56% 357/638 [00:07<00:05, 48.71it/s, dev_acc=0.78, epoch=6, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  57% 362/638 [00:08<00:06, 42.85it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  58% 368/638 [00:08<00:05, 45.31it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  58% 373/638 [00:08<00:06, 38.74it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  59% 379/638 [00:08<00:06, 42.16it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  60% 384/638 [00:08<00:06, 39.40it/s, dev_acc=0.78, epoch=6, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  61% 390/638 [00:08<00:05, 42.70it/s, dev_acc=0.78, epoch=6, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  62% 396/638 [00:08<00:05, 45.50it/s, dev_acc=0.78, epoch=6, loss=0.13, train_acc=0.85]\u001b[A\n",
            "Batches:  63% 401/638 [00:09<00:05, 41.53it/s, dev_acc=0.78, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  64% 407/638 [00:09<00:05, 43.86it/s, dev_acc=0.78, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  65% 413/638 [00:09<00:04, 47.15it/s, dev_acc=0.78, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  66% 419/638 [00:09<00:04, 49.36it/s, dev_acc=0.78, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  67% 425/638 [00:09<00:04, 44.93it/s, dev_acc=0.79, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  68% 431/638 [00:09<00:04, 46.84it/s, dev_acc=0.79, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  68% 437/638 [00:09<00:04, 49.08it/s, dev_acc=0.79, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  69% 443/638 [00:09<00:04, 44.25it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  70% 449/638 [00:10<00:04, 46.83it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  71% 454/638 [00:10<00:03, 46.69it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  72% 460/638 [00:10<00:03, 48.22it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  73% 465/638 [00:10<00:04, 42.70it/s, dev_acc=0.78, epoch=6, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  74% 471/638 [00:10<00:03, 45.24it/s, dev_acc=0.78, epoch=6, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  75% 476/638 [00:10<00:04, 37.61it/s, dev_acc=0.78, epoch=6, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  75% 481/638 [00:10<00:04, 36.40it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  76% 487/638 [00:10<00:03, 40.03it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  77% 493/638 [00:11<00:03, 42.71it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  78% 498/638 [00:11<00:03, 44.63it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  79% 503/638 [00:11<00:03, 41.10it/s, dev_acc=0.78, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  80% 509/638 [00:11<00:02, 43.46it/s, dev_acc=0.78, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  81% 515/638 [00:11<00:02, 45.41it/s, dev_acc=0.78, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  82% 520/638 [00:11<00:02, 46.66it/s, dev_acc=0.78, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  82% 525/638 [00:11<00:02, 42.24it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  83% 531/638 [00:11<00:02, 44.23it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  84% 537/638 [00:12<00:02, 46.23it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  85% 542/638 [00:12<00:02, 41.23it/s, dev_acc=0.79, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  86% 548/638 [00:12<00:02, 43.93it/s, dev_acc=0.79, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  87% 554/638 [00:12<00:01, 46.10it/s, dev_acc=0.79, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  88% 560/638 [00:12<00:01, 47.58it/s, dev_acc=0.79, epoch=6, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  89% 565/638 [00:12<00:01, 42.54it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  89% 571/638 [00:12<00:01, 45.65it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  90% 576/638 [00:13<00:01, 38.14it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  91% 581/638 [00:13<00:01, 36.80it/s, dev_acc=0.78, epoch=6, loss=0.14, train_acc=0.86]\u001b[A\n",
            "Batches:  92% 587/638 [00:13<00:01, 40.84it/s, dev_acc=0.78, epoch=6, loss=0.14, train_acc=0.86]\u001b[A\n",
            "Batches:  93% 592/638 [00:13<00:01, 42.93it/s, dev_acc=0.78, epoch=6, loss=0.14, train_acc=0.86]\u001b[A\n",
            "Batches:  94% 598/638 [00:13<00:00, 45.49it/s, dev_acc=0.78, epoch=6, loss=0.14, train_acc=0.86]\u001b[A\n",
            "Batches:  95% 603/638 [00:13<00:00, 41.67it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  95% 609/638 [00:13<00:00, 44.27it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  96% 615/638 [00:13<00:00, 45.89it/s, dev_acc=0.78, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  97% 621/638 [00:14<00:00, 42.63it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  98% 627/638 [00:14<00:00, 44.84it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  99% 633/638 [00:14<00:00, 46.58it/s, dev_acc=0.79, epoch=6, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Epochs:  70% 7/10 [01:50<00:46, 15.64s/it]\n",
            "Batches:   0% 0/638 [00:00<?, ?it/s, acc=0]\u001b[A\n",
            "Batches:   0% 3/638 [00:00<00:23, 26.75it/s, dev_acc=0.79, epoch=7, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:   1% 9/638 [00:00<00:19, 31.53it/s, dev_acc=0.79, epoch=7, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:   2% 15/638 [00:00<00:17, 35.68it/s, dev_acc=0.79, epoch=7, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:   3% 21/638 [00:00<00:17, 35.95it/s, dev_acc=0.78, epoch=7, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:   4% 27/638 [00:00<00:15, 39.81it/s, dev_acc=0.78, epoch=7, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:   5% 33/638 [00:00<00:14, 42.96it/s, dev_acc=0.78, epoch=7, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:   6% 38/638 [00:01<00:19, 30.78it/s, dev_acc=0.78, epoch=7, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:   7% 42/638 [00:01<00:19, 30.91it/s, dev_acc=0.78, epoch=7, loss=0.10, train_acc=0.86]\u001b[A\n",
            "Batches:   8% 48/638 [00:01<00:16, 35.17it/s, dev_acc=0.78, epoch=7, loss=0.10, train_acc=0.86]\u001b[A\n",
            "Batches:   8% 54/638 [00:01<00:14, 39.15it/s, dev_acc=0.78, epoch=7, loss=0.10, train_acc=0.86]\u001b[A\n",
            "Batches:   9% 60/638 [00:01<00:13, 42.12it/s, dev_acc=0.78, epoch=7, loss=0.10, train_acc=0.86]\u001b[A\n",
            "Batches:  10% 65/638 [00:01<00:14, 39.17it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  11% 71/638 [00:01<00:13, 42.74it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  12% 77/638 [00:01<00:12, 45.17it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  13% 82/638 [00:01<00:13, 41.40it/s, dev_acc=0.78, epoch=7, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  14% 88/638 [00:02<00:12, 44.58it/s, dev_acc=0.78, epoch=7, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  15% 94/638 [00:02<00:11, 47.22it/s, dev_acc=0.78, epoch=7, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  16% 100/638 [00:02<00:10, 49.12it/s, dev_acc=0.78, epoch=7, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  17% 106/638 [00:02<00:12, 44.25it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  18% 112/638 [00:02<00:11, 46.67it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  18% 118/638 [00:02<00:10, 47.90it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  19% 123/638 [00:02<00:12, 42.07it/s, dev_acc=0.79, epoch=7, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  20% 129/638 [00:02<00:11, 44.55it/s, dev_acc=0.79, epoch=7, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  21% 134/638 [00:03<00:13, 37.11it/s, dev_acc=0.79, epoch=7, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  22% 140/638 [00:03<00:12, 40.76it/s, dev_acc=0.79, epoch=7, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  23% 145/638 [00:03<00:12, 38.40it/s, dev_acc=0.79, epoch=7, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  24% 151/638 [00:03<00:11, 41.75it/s, dev_acc=0.79, epoch=7, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  25% 157/638 [00:03<00:10, 44.21it/s, dev_acc=0.79, epoch=7, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  25% 162/638 [00:03<00:11, 40.01it/s, dev_acc=0.79, epoch=7, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  26% 168/638 [00:03<00:10, 42.90it/s, dev_acc=0.79, epoch=7, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  27% 174/638 [00:04<00:10, 45.15it/s, dev_acc=0.79, epoch=7, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  28% 180/638 [00:04<00:09, 47.15it/s, dev_acc=0.79, epoch=7, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  29% 185/638 [00:04<00:10, 42.21it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  30% 191/638 [00:04<00:10, 44.52it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  31% 197/638 [00:04<00:09, 46.47it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  32% 202/638 [00:04<00:10, 41.95it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  33% 208/638 [00:04<00:09, 44.13it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  34% 214/638 [00:04<00:09, 46.31it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  34% 220/638 [00:05<00:08, 47.90it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  35% 225/638 [00:05<00:09, 42.86it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  36% 231/638 [00:05<00:09, 44.98it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  37% 236/638 [00:05<00:10, 38.71it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  38% 241/638 [00:05<00:10, 37.25it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  39% 247/638 [00:05<00:09, 41.37it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  40% 253/638 [00:05<00:08, 44.17it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  41% 259/638 [00:05<00:08, 46.39it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  41% 264/638 [00:06<00:09, 40.02it/s, dev_acc=0.79, epoch=7, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  42% 270/638 [00:06<00:08, 43.21it/s, dev_acc=0.79, epoch=7, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  43% 276/638 [00:06<00:07, 45.45it/s, dev_acc=0.79, epoch=7, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  44% 281/638 [00:06<00:08, 41.14it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  45% 287/638 [00:06<00:07, 43.91it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  46% 293/638 [00:06<00:07, 46.52it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  47% 299/638 [00:06<00:07, 47.80it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  48% 304/638 [00:06<00:07, 42.95it/s, dev_acc=0.78, epoch=7, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  49% 310/638 [00:07<00:07, 45.78it/s, dev_acc=0.78, epoch=7, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  50% 316/638 [00:07<00:06, 47.81it/s, dev_acc=0.78, epoch=7, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  50% 321/638 [00:07<00:07, 42.95it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  51% 327/638 [00:07<00:06, 45.65it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  52% 333/638 [00:07<00:06, 47.13it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  53% 338/638 [00:07<00:07, 39.17it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  54% 343/638 [00:07<00:07, 37.59it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  55% 349/638 [00:08<00:07, 41.26it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  56% 355/638 [00:08<00:06, 43.89it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  57% 361/638 [00:08<00:06, 41.45it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  58% 367/638 [00:08<00:06, 44.10it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  58% 373/638 [00:08<00:05, 46.12it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  59% 379/638 [00:08<00:05, 47.84it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  60% 384/638 [00:08<00:05, 42.70it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  61% 390/638 [00:08<00:05, 45.86it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  62% 396/638 [00:09<00:05, 48.28it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  63% 402/638 [00:09<00:05, 44.32it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  64% 408/638 [00:09<00:04, 46.09it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  65% 414/638 [00:09<00:04, 47.69it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  66% 420/638 [00:09<00:04, 49.67it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  67% 426/638 [00:09<00:04, 45.45it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  68% 432/638 [00:09<00:04, 47.11it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  68% 437/638 [00:09<00:05, 39.49it/s, dev_acc=0.78, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  69% 442/638 [00:10<00:05, 37.82it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  70% 448/638 [00:10<00:04, 41.90it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  71% 454/638 [00:10<00:04, 44.58it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  72% 460/638 [00:10<00:03, 46.67it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  73% 465/638 [00:10<00:04, 42.64it/s, dev_acc=0.79, epoch=7, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  74% 471/638 [00:10<00:03, 45.09it/s, dev_acc=0.79, epoch=7, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  75% 477/638 [00:10<00:03, 47.50it/s, dev_acc=0.79, epoch=7, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  76% 482/638 [00:10<00:03, 42.46it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  76% 488/638 [00:11<00:03, 45.23it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  77% 494/638 [00:11<00:03, 46.59it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  78% 500/638 [00:11<00:02, 47.87it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  79% 505/638 [00:11<00:03, 42.80it/s, dev_acc=0.79, epoch=7, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  80% 511/638 [00:11<00:02, 45.27it/s, dev_acc=0.79, epoch=7, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  81% 517/638 [00:11<00:02, 47.41it/s, dev_acc=0.79, epoch=7, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  82% 522/638 [00:11<00:02, 42.44it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  83% 528/638 [00:11<00:02, 45.38it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  84% 534/638 [00:12<00:02, 39.15it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  85% 540/638 [00:12<00:02, 42.63it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  85% 545/638 [00:12<00:02, 40.24it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  86% 551/638 [00:12<00:01, 44.01it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  87% 557/638 [00:12<00:01, 46.74it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  88% 562/638 [00:12<00:01, 42.44it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  89% 568/638 [00:12<00:01, 45.14it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  90% 574/638 [00:12<00:01, 47.35it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  91% 580/638 [00:13<00:01, 49.29it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  92% 586/638 [00:13<00:01, 44.87it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  93% 592/638 [00:13<00:00, 47.26it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  94% 598/638 [00:13<00:00, 49.01it/s, dev_acc=0.79, epoch=7, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  95% 604/638 [00:13<00:00, 44.78it/s, dev_acc=0.78, epoch=7, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  96% 610/638 [00:13<00:00, 47.87it/s, dev_acc=0.78, epoch=7, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  97% 616/638 [00:13<00:00, 50.02it/s, dev_acc=0.78, epoch=7, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  97% 622/638 [00:14<00:00, 45.42it/s, dev_acc=0.79, epoch=7, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  98% 628/638 [00:14<00:00, 48.03it/s, dev_acc=0.79, epoch=7, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Batches:  99% 634/638 [00:14<00:00, 41.20it/s, dev_acc=0.79, epoch=7, loss=0.13, train_acc=0.86]\u001b[A\n",
            "Epochs:  80% 8/10 [02:05<00:30, 15.41s/it]\n",
            "Batches:   0% 0/638 [00:00<?, ?it/s, acc=0]\u001b[A\n",
            "Batches:   0% 3/638 [00:00<00:24, 25.80it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:   1% 9/638 [00:00<00:20, 30.40it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:   2% 15/638 [00:00<00:17, 34.64it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:   3% 21/638 [00:00<00:17, 35.26it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:   4% 26/638 [00:00<00:16, 38.05it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:   5% 32/638 [00:00<00:14, 42.08it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:   6% 38/638 [00:00<00:13, 44.29it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:   7% 43/638 [00:01<00:15, 39.38it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:   8% 49/638 [00:01<00:13, 42.22it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:   9% 55/638 [00:01<00:13, 44.58it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:   9% 60/638 [00:01<00:12, 44.84it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  10% 65/638 [00:01<00:14, 39.90it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  11% 70/638 [00:01<00:13, 41.69it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  12% 75/638 [00:01<00:13, 42.86it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  13% 81/638 [00:01<00:13, 39.90it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  13% 86/638 [00:02<00:13, 40.88it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  14% 92/638 [00:02<00:12, 42.99it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  15% 97/638 [00:02<00:15, 35.79it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  16% 101/638 [00:02<00:16, 32.38it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  17% 107/638 [00:02<00:14, 36.25it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  18% 112/638 [00:02<00:13, 38.46it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  18% 118/638 [00:02<00:12, 41.26it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  19% 123/638 [00:02<00:13, 38.35it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  20% 129/638 [00:03<00:12, 41.48it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  21% 135/638 [00:03<00:11, 43.70it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  22% 140/638 [00:03<00:10, 45.35it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  23% 145/638 [00:03<00:12, 39.70it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  24% 150/638 [00:03<00:11, 42.31it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  24% 155/638 [00:03<00:11, 43.59it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  25% 160/638 [00:03<00:10, 44.80it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.86]\u001b[A\n",
            "Batches:  26% 165/638 [00:03<00:11, 40.57it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  27% 171/638 [00:04<00:10, 43.32it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  28% 177/638 [00:04<00:10, 45.70it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  29% 182/638 [00:04<00:11, 40.54it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  29% 187/638 [00:04<00:10, 42.97it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  30% 192/638 [00:04<00:10, 44.43it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  31% 197/638 [00:04<00:11, 36.95it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  32% 202/638 [00:04<00:12, 35.25it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  33% 208/638 [00:05<00:11, 37.42it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  33% 213/638 [00:05<00:10, 40.27it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  34% 218/638 [00:05<00:09, 42.48it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  35% 223/638 [00:05<00:10, 38.63it/s, dev_acc=0.78, epoch=8, loss=0.10, train_acc=0.87]\u001b[A\n",
            "Batches:  36% 228/638 [00:05<00:09, 41.06it/s, dev_acc=0.78, epoch=8, loss=0.10, train_acc=0.87]\u001b[A\n",
            "Batches:  37% 233/638 [00:05<00:09, 42.71it/s, dev_acc=0.78, epoch=8, loss=0.10, train_acc=0.87]\u001b[A\n",
            "Batches:  37% 238/638 [00:05<00:09, 43.13it/s, dev_acc=0.78, epoch=8, loss=0.10, train_acc=0.87]\u001b[A\n",
            "Batches:  38% 243/638 [00:05<00:10, 38.24it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  39% 248/638 [00:05<00:09, 40.69it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  40% 253/638 [00:06<00:09, 42.29it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  40% 258/638 [00:06<00:08, 43.98it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  41% 263/638 [00:06<00:09, 39.94it/s, dev_acc=0.79, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  42% 268/638 [00:06<00:08, 41.62it/s, dev_acc=0.79, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  43% 273/638 [00:06<00:08, 42.72it/s, dev_acc=0.79, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  44% 278/638 [00:06<00:08, 43.55it/s, dev_acc=0.79, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  44% 283/638 [00:06<00:09, 38.67it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  45% 288/638 [00:06<00:08, 40.70it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  46% 293/638 [00:07<00:08, 42.30it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  47% 298/638 [00:07<00:09, 34.89it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  47% 302/638 [00:07<00:09, 33.94it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.88]\u001b[A\n",
            "Batches:  48% 308/638 [00:07<00:08, 37.87it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.88]\u001b[A\n",
            "Batches:  49% 313/638 [00:07<00:07, 40.79it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.88]\u001b[A\n",
            "Batches:  50% 319/638 [00:07<00:07, 43.64it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.88]\u001b[A\n",
            "Batches:  51% 324/638 [00:07<00:08, 39.16it/s, dev_acc=0.79, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  52% 330/638 [00:07<00:07, 42.54it/s, dev_acc=0.79, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  53% 336/638 [00:08<00:06, 44.12it/s, dev_acc=0.79, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  53% 341/638 [00:08<00:07, 39.08it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  54% 347/638 [00:08<00:06, 41.95it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  55% 352/638 [00:08<00:06, 43.94it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  56% 357/638 [00:08<00:06, 45.04it/s, dev_acc=0.78, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  57% 362/638 [00:08<00:06, 40.30it/s, dev_acc=0.79, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  58% 367/638 [00:08<00:06, 42.40it/s, dev_acc=0.79, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  58% 373/638 [00:08<00:05, 44.70it/s, dev_acc=0.79, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  59% 378/638 [00:09<00:05, 45.21it/s, dev_acc=0.79, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  60% 383/638 [00:09<00:06, 40.16it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  61% 389/638 [00:09<00:05, 43.10it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  62% 394/638 [00:09<00:05, 44.77it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  63% 399/638 [00:09<00:06, 37.76it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  63% 404/638 [00:09<00:06, 35.60it/s, dev_acc=0.78, epoch=8, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  64% 409/638 [00:09<00:05, 38.39it/s, dev_acc=0.78, epoch=8, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  65% 414/638 [00:09<00:05, 40.40it/s, dev_acc=0.78, epoch=8, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  66% 419/638 [00:10<00:05, 41.95it/s, dev_acc=0.78, epoch=8, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  66% 424/638 [00:10<00:06, 35.34it/s, dev_acc=0.79, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  67% 429/638 [00:10<00:05, 37.63it/s, dev_acc=0.79, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  68% 434/638 [00:10<00:05, 39.74it/s, dev_acc=0.79, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  69% 440/638 [00:10<00:04, 42.84it/s, dev_acc=0.79, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  70% 445/638 [00:10<00:04, 38.91it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  71% 451/638 [00:10<00:04, 41.82it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  71% 456/638 [00:11<00:04, 43.59it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  72% 461/638 [00:11<00:04, 39.53it/s, dev_acc=0.78, epoch=8, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  73% 466/638 [00:11<00:04, 41.37it/s, dev_acc=0.78, epoch=8, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  74% 471/638 [00:11<00:03, 42.87it/s, dev_acc=0.78, epoch=8, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  75% 476/638 [00:11<00:03, 43.96it/s, dev_acc=0.78, epoch=8, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  75% 481/638 [00:11<00:04, 38.90it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  76% 486/638 [00:11<00:03, 40.22it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  77% 491/638 [00:11<00:03, 41.42it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  78% 496/638 [00:12<00:03, 36.00it/s, dev_acc=0.78, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  79% 501/638 [00:12<00:04, 34.21it/s, dev_acc=0.79, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  79% 506/638 [00:12<00:03, 37.51it/s, dev_acc=0.79, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  80% 511/638 [00:12<00:03, 40.38it/s, dev_acc=0.79, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  81% 516/638 [00:12<00:02, 42.45it/s, dev_acc=0.79, epoch=8, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  82% 521/638 [00:12<00:03, 38.44it/s, dev_acc=0.79, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  82% 526/638 [00:12<00:02, 40.95it/s, dev_acc=0.79, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  83% 531/638 [00:12<00:02, 42.51it/s, dev_acc=0.79, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  84% 537/638 [00:13<00:02, 45.10it/s, dev_acc=0.79, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  85% 542/638 [00:13<00:02, 40.88it/s, dev_acc=0.79, epoch=8, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  86% 547/638 [00:13<00:02, 42.18it/s, dev_acc=0.79, epoch=8, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  87% 552/638 [00:13<00:01, 43.73it/s, dev_acc=0.79, epoch=8, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  87% 558/638 [00:13<00:01, 45.54it/s, dev_acc=0.79, epoch=8, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  88% 563/638 [00:13<00:01, 40.59it/s, dev_acc=0.79, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  89% 569/638 [00:13<00:01, 42.62it/s, dev_acc=0.79, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  90% 574/638 [00:13<00:01, 43.91it/s, dev_acc=0.79, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  91% 579/638 [00:13<00:01, 45.57it/s, dev_acc=0.79, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  92% 584/638 [00:14<00:01, 41.07it/s, dev_acc=0.79, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  92% 589/638 [00:14<00:01, 43.36it/s, dev_acc=0.79, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  93% 594/638 [00:14<00:00, 44.71it/s, dev_acc=0.79, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  94% 599/638 [00:14<00:01, 37.25it/s, dev_acc=0.79, epoch=8, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  95% 604/638 [00:14<00:00, 35.04it/s, dev_acc=0.78, epoch=8, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  95% 609/638 [00:14<00:00, 37.57it/s, dev_acc=0.78, epoch=8, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  96% 614/638 [00:14<00:00, 39.80it/s, dev_acc=0.78, epoch=8, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  97% 619/638 [00:15<00:00, 41.76it/s, dev_acc=0.78, epoch=8, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  98% 624/638 [00:15<00:00, 37.57it/s, dev_acc=0.78, epoch=8, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  99% 629/638 [00:15<00:00, 38.81it/s, dev_acc=0.78, epoch=8, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  99% 634/638 [00:15<00:00, 38.33it/s, dev_acc=0.78, epoch=8, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Epochs:  90% 9/10 [02:21<00:15, 15.57s/it]\n",
            "Batches:   0% 0/638 [00:00<?, ?it/s, acc=0]\u001b[A\n",
            "Batches:   0% 2/638 [00:00<00:32, 19.60it/s, dev_acc=0.79, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:   1% 8/638 [00:00<00:26, 24.04it/s, dev_acc=0.79, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:   2% 13/638 [00:00<00:22, 27.87it/s, dev_acc=0.79, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:   3% 18/638 [00:00<00:19, 31.89it/s, dev_acc=0.79, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:   3% 22/638 [00:00<00:19, 31.39it/s, dev_acc=0.79, epoch=9, loss=0.10, train_acc=0.87]\u001b[A\n",
            "Batches:   4% 27/638 [00:00<00:17, 34.87it/s, dev_acc=0.79, epoch=9, loss=0.10, train_acc=0.87]\u001b[A\n",
            "Batches:   5% 32/638 [00:00<00:15, 38.18it/s, dev_acc=0.79, epoch=9, loss=0.10, train_acc=0.87]\u001b[A\n",
            "Batches:   6% 37/638 [00:00<00:14, 40.78it/s, dev_acc=0.79, epoch=9, loss=0.10, train_acc=0.87]\u001b[A\n",
            "Batches:   7% 42/638 [00:01<00:15, 37.72it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:   7% 47/638 [00:01<00:14, 40.40it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:   8% 53/638 [00:01<00:13, 42.85it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:   9% 58/638 [00:01<00:15, 36.72it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  10% 62/638 [00:01<00:16, 34.78it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.88]\u001b[A\n",
            "Batches:  11% 67/638 [00:01<00:14, 38.11it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.88]\u001b[A\n",
            "Batches:  11% 73/638 [00:01<00:13, 41.05it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.88]\u001b[A\n",
            "Batches:  12% 79/638 [00:01<00:12, 43.68it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.88]\u001b[A\n",
            "Batches:  13% 84/638 [00:02<00:13, 39.74it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  14% 90/638 [00:02<00:12, 42.21it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  15% 96/638 [00:02<00:12, 44.51it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  16% 101/638 [00:02<00:13, 40.19it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  17% 107/638 [00:02<00:12, 42.73it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  18% 113/638 [00:02<00:11, 44.92it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  19% 119/638 [00:02<00:11, 46.37it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  19% 124/638 [00:02<00:12, 41.42it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  20% 130/638 [00:03<00:11, 43.85it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  21% 136/638 [00:03<00:10, 46.26it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  22% 141/638 [00:03<00:12, 40.83it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  23% 146/638 [00:03<00:11, 42.89it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  24% 152/638 [00:03<00:10, 44.91it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  25% 158/638 [00:03<00:12, 39.21it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  26% 163/638 [00:03<00:12, 36.98it/s, dev_acc=0.78, epoch=9, loss=0.10, train_acc=0.87]\u001b[A\n",
            "Batches:  26% 168/638 [00:04<00:11, 39.79it/s, dev_acc=0.78, epoch=9, loss=0.10, train_acc=0.87]\u001b[A\n",
            "Batches:  27% 174/638 [00:04<00:10, 42.49it/s, dev_acc=0.78, epoch=9, loss=0.10, train_acc=0.87]\u001b[A\n",
            "Batches:  28% 179/638 [00:04<00:10, 44.47it/s, dev_acc=0.78, epoch=9, loss=0.10, train_acc=0.87]\u001b[A\n",
            "Batches:  29% 184/638 [00:04<00:11, 40.45it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  30% 190/638 [00:04<00:10, 43.94it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  31% 196/638 [00:04<00:09, 46.27it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  32% 201/638 [00:04<00:10, 41.09it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  32% 206/638 [00:04<00:09, 43.22it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  33% 212/638 [00:04<00:09, 45.31it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  34% 218/638 [00:05<00:08, 47.10it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  35% 223/638 [00:05<00:09, 41.65it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  36% 229/638 [00:05<00:09, 44.12it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  37% 235/638 [00:05<00:08, 45.90it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  38% 240/638 [00:05<00:08, 46.73it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  38% 245/638 [00:05<00:09, 41.31it/s, dev_acc=0.78, epoch=9, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  39% 250/638 [00:05<00:08, 43.27it/s, dev_acc=0.78, epoch=9, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  40% 256/638 [00:05<00:08, 45.43it/s, dev_acc=0.78, epoch=9, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  41% 261/638 [00:06<00:11, 33.86it/s, dev_acc=0.79, epoch=9, loss=0.10, train_acc=0.87]\u001b[A\n",
            "Batches:  42% 266/638 [00:06<00:10, 37.19it/s, dev_acc=0.79, epoch=9, loss=0.10, train_acc=0.87]\u001b[A\n",
            "Batches:  43% 272/638 [00:06<00:09, 40.11it/s, dev_acc=0.79, epoch=9, loss=0.10, train_acc=0.87]\u001b[A\n",
            "Batches:  44% 278/638 [00:06<00:08, 42.95it/s, dev_acc=0.79, epoch=9, loss=0.10, train_acc=0.87]\u001b[A\n",
            "Batches:  44% 283/638 [00:06<00:08, 39.56it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  45% 289/638 [00:06<00:08, 42.18it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  46% 294/638 [00:06<00:07, 44.02it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  47% 299/638 [00:07<00:07, 43.24it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  48% 304/638 [00:07<00:08, 39.22it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  48% 309/638 [00:07<00:07, 41.54it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  49% 315/638 [00:07<00:07, 43.64it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.86]\u001b[A\n",
            "Batches:  50% 321/638 [00:07<00:07, 40.67it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  51% 327/638 [00:07<00:07, 43.27it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  52% 333/638 [00:07<00:06, 45.72it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  53% 339/638 [00:07<00:06, 47.20it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  54% 344/638 [00:08<00:07, 41.80it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  55% 349/638 [00:08<00:06, 43.96it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  56% 355/638 [00:08<00:06, 45.86it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  56% 360/638 [00:08<00:07, 35.08it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  57% 364/638 [00:08<00:08, 33.05it/s, dev_acc=0.78, epoch=9, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  58% 370/638 [00:08<00:07, 36.92it/s, dev_acc=0.78, epoch=9, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  59% 376/638 [00:08<00:06, 40.25it/s, dev_acc=0.78, epoch=9, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  60% 381/638 [00:09<00:06, 37.13it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.88]\u001b[A\n",
            "Batches:  61% 386/638 [00:09<00:06, 40.08it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.88]\u001b[A\n",
            "Batches:  61% 391/638 [00:09<00:05, 42.60it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.88]\u001b[A\n",
            "Batches:  62% 396/638 [00:09<00:05, 44.35it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.88]\u001b[A\n",
            "Batches:  63% 401/638 [00:09<00:06, 39.09it/s, dev_acc=0.79, epoch=9, loss=0.12, train_acc=0.88]\u001b[A\n",
            "Batches:  64% 406/638 [00:09<00:05, 41.42it/s, dev_acc=0.79, epoch=9, loss=0.12, train_acc=0.88]\u001b[A\n",
            "Batches:  64% 411/638 [00:09<00:05, 42.95it/s, dev_acc=0.79, epoch=9, loss=0.12, train_acc=0.88]\u001b[A\n",
            "Batches:  65% 416/638 [00:09<00:04, 44.62it/s, dev_acc=0.79, epoch=9, loss=0.12, train_acc=0.88]\u001b[A\n",
            "Batches:  66% 421/638 [00:09<00:05, 39.88it/s, dev_acc=0.78, epoch=9, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  67% 426/638 [00:10<00:05, 42.12it/s, dev_acc=0.78, epoch=9, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  68% 431/638 [00:10<00:04, 43.88it/s, dev_acc=0.78, epoch=9, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  68% 436/638 [00:10<00:04, 44.60it/s, dev_acc=0.78, epoch=9, loss=0.13, train_acc=0.87]\u001b[A\n",
            "Batches:  69% 441/638 [00:10<00:04, 39.98it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  70% 446/638 [00:10<00:04, 41.91it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  71% 451/638 [00:10<00:04, 43.41it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  71% 456/638 [00:10<00:04, 44.73it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  72% 461/638 [00:11<00:05, 32.12it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.88]\u001b[A\n",
            "Batches:  73% 467/638 [00:11<00:04, 36.03it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.88]\u001b[A\n",
            "Batches:  74% 473/638 [00:11<00:04, 39.54it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.88]\u001b[A\n",
            "Batches:  75% 478/638 [00:11<00:03, 41.78it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.88]\u001b[A\n",
            "Batches:  76% 483/638 [00:11<00:04, 38.40it/s, dev_acc=0.79, epoch=9, loss=0.10, train_acc=0.88]\u001b[A\n",
            "Batches:  77% 489/638 [00:11<00:03, 41.58it/s, dev_acc=0.79, epoch=9, loss=0.10, train_acc=0.88]\u001b[A\n",
            "Batches:  77% 494/638 [00:11<00:03, 43.75it/s, dev_acc=0.79, epoch=9, loss=0.10, train_acc=0.88]\u001b[A\n",
            "Batches:  78% 499/638 [00:11<00:03, 45.08it/s, dev_acc=0.79, epoch=9, loss=0.10, train_acc=0.88]\u001b[A\n",
            "Batches:  79% 504/638 [00:12<00:03, 40.51it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  80% 509/638 [00:12<00:03, 42.45it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  81% 515/638 [00:12<00:02, 44.54it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  82% 520/638 [00:12<00:02, 45.32it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  82% 525/638 [00:12<00:02, 40.75it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  83% 530/638 [00:12<00:02, 42.56it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  84% 535/638 [00:12<00:02, 44.06it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  85% 540/638 [00:12<00:02, 45.39it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.87]\u001b[A\n",
            "Batches:  85% 545/638 [00:12<00:02, 40.75it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.88]\u001b[A\n",
            "Batches:  86% 551/638 [00:13<00:02, 43.37it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.88]\u001b[A\n",
            "Batches:  87% 557/638 [00:13<00:01, 45.44it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.88]\u001b[A\n",
            "Batches:  88% 562/638 [00:13<00:02, 34.52it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  89% 568/638 [00:13<00:01, 38.23it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  90% 573/638 [00:13<00:01, 40.24it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  91% 578/638 [00:13<00:01, 42.57it/s, dev_acc=0.78, epoch=9, loss=0.11, train_acc=0.87]\u001b[A\n",
            "Batches:  91% 583/638 [00:13<00:01, 39.44it/s, dev_acc=0.79, epoch=9, loss=0.12, train_acc=0.88]\u001b[A\n",
            "Batches:  92% 589/638 [00:13<00:01, 43.06it/s, dev_acc=0.79, epoch=9, loss=0.12, train_acc=0.88]\u001b[A\n",
            "Batches:  93% 595/638 [00:14<00:00, 45.59it/s, dev_acc=0.79, epoch=9, loss=0.12, train_acc=0.88]\u001b[A\n",
            "Batches:  94% 601/638 [00:14<00:00, 42.00it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.88]\u001b[A\n",
            "Batches:  95% 607/638 [00:14<00:00, 44.51it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.88]\u001b[A\n",
            "Batches:  96% 613/638 [00:14<00:00, 46.55it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.88]\u001b[A\n",
            "Batches:  97% 619/638 [00:14<00:00, 48.68it/s, dev_acc=0.78, epoch=9, loss=0.12, train_acc=0.88]\u001b[A\n",
            "Batches:  98% 625/638 [00:14<00:00, 44.50it/s, dev_acc=0.79, epoch=9, loss=0.12, train_acc=0.88]\u001b[A\n",
            "Batches:  99% 630/638 [00:14<00:00, 45.67it/s, dev_acc=0.79, epoch=9, loss=0.12, train_acc=0.88]\u001b[A\n",
            "Batches: 100% 636/638 [00:15<00:00, 47.65it/s, dev_acc=0.79, epoch=9, loss=0.12, train_acc=0.88]\u001b[A\n",
            "Epochs: 100% 10/10 [02:37<00:00, 15.56s/it]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DdMoyptoFum2",
        "colab_type": "code",
        "outputId": "c18ff581-f15d-4591-a826-d9ca8949c57c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "!cd multihead-siamese-nets && python3 run.py train rnn SNLI"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tflearn/helpers/summarizer.py:9: The name tf.summary.merge is deprecated. Please use tf.compat.v1.summary.merge instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tflearn/helpers/trainer.py:25: The name tf.summary.FileWriter is deprecated. Please use tf.compat.v1.summary.FileWriter instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tflearn/collections.py:13: The name tf.GraphKeys is deprecated. Please use tf.compat.v1.GraphKeys instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tflearn/config.py:123: The name tf.get_collection is deprecated. Please use tf.compat.v1.get_collection instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tflearn/config.py:129: The name tf.add_to_collection is deprecated. Please use tf.compat.v1.add_to_collection instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tflearn/config.py:131: The name tf.assign is deprecated. Please use tf.compat.v1.assign instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/utils/data_utils.py:7: The name tf.logging.info is deprecated. Please use tf.compat.v1.logging.info instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/utils/other_utils.py:10: The name tf.logging.set_verbosity is deprecated. Please use tf.compat.v1.logging.set_verbosity instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/utils/other_utils.py:10: The name tf.logging.INFO is deprecated. Please use tf.compat.v1.logging.INFO instead.\n",
            "\n",
            "INFO:tensorflow:Setting visible GPU to 0\n",
            "INFO:tensorflow:Reading main configuration.\n",
            "INFO:tensorflow:Reading configuration for rnn model.\n",
            "INFO:tensorflow:Chosen word embeddings.\n",
            "INFO:tensorflow:Maximum sentence length : 78\n",
            "INFO:tensorflow:Processing sentences with word embeddings...\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tflearn/data_utils.py:201: VocabularyProcessor.__init__ (from tensorflow.contrib.learn.python.learn.preprocessing.text) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tensorflow/transform or tf.data.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/contrib/learn/python/learn/preprocessing/text.py:154: CategoricalVocabulary.__init__ (from tensorflow.contrib.learn.python.learn.preprocessing.categorical_vocabulary) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tensorflow/transform or tf.data.\n",
            "INFO:tensorflow:Sentences have been successfully processed.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/contrib/learn/python/learn/preprocessing/text.py:170: tokenizer (from tensorflow.contrib.learn.python.learn.preprocessing.text) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tensorflow/transform or tf.data.\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/models/base_model.py:15: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/models/base_model.py:28: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/models/base_model.py:29: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/layers/recurrent.py:30: GRUCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "This class is equivalent as tf.keras.layers.GRUCell, and will be replaced by that in Tensorflow 2.0.\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/layers/recurrent.py:45: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn_cell_impl.py:559: Layer.add_variable (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `layer.add_weight` method instead.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn_cell_impl.py:565: calling Constant.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn_cell_impl.py:575: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/layers/losses.py:55: The name tf.losses.mean_squared_error is deprecated. Please use tf.compat.v1.losses.mean_squared_error instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/losses/losses_impl.py:121: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/layers/basics.py:66: The name tf.train.AdamOptimizer is deprecated. Please use tf.compat.v1.train.AdamOptimizer instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/models/base_model.py:42: The name tf.rint is deprecated. Please use tf.math.rint instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/models/base_model.py:43: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.cast` instead.\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/models/base_model.py:47: The name tf.summary.scalar is deprecated. Please use tf.compat.v1.summary.scalar instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/models/base_model.py:49: The name tf.summary.merge_all is deprecated. Please use tf.compat.v1.summary.merge_all instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/multihead-siamese-nets/utils/model_saver.py:9: The name tf.train.Saver is deprecated. Please use tf.compat.v1.train.Saver instead.\n",
            "\n",
            "WARNING:tensorflow:From run.py:73: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
            "\n",
            "WARNING:tensorflow:From run.py:78: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
            "\n",
            "2019-12-19 22:04:36.259656: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
            "2019-12-19 22:04:36.263939: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2200000000 Hz\n",
            "2019-12-19 22:04:36.264122: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x2ebd480 initialized for platform Host (this does not guarantee that XLA will be used). Devices:\n",
            "2019-12-19 22:04:36.264149: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version\n",
            "2019-12-19 22:04:36.266104: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1\n",
            "2019-12-19 22:04:36.364513: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2019-12-19 22:04:36.365255: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x2ebd2c0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
            "2019-12-19 22:04:36.365288: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Tesla P100-PCIE-16GB, Compute Capability 6.0\n",
            "2019-12-19 22:04:36.365463: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2019-12-19 22:04:36.366024: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 0 with properties: \n",
            "name: Tesla P100-PCIE-16GB major: 6 minor: 0 memoryClockRate(GHz): 1.3285\n",
            "pciBusID: 0000:00:04.0\n",
            "2019-12-19 22:04:36.366359: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0\n",
            "2019-12-19 22:04:36.367657: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0\n",
            "2019-12-19 22:04:36.368817: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10.0\n",
            "2019-12-19 22:04:36.369150: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10.0\n",
            "2019-12-19 22:04:36.370473: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10.0\n",
            "2019-12-19 22:04:36.371419: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10.0\n",
            "2019-12-19 22:04:36.374607: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7\n",
            "2019-12-19 22:04:36.374712: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2019-12-19 22:04:36.375273: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2019-12-19 22:04:36.375828: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Adding visible gpu devices: 0\n",
            "2019-12-19 22:04:36.375892: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0\n",
            "2019-12-19 22:04:36.376996: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1159] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
            "2019-12-19 22:04:36.377022: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1165]      0 \n",
            "2019-12-19 22:04:36.377032: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1178] 0:   N \n",
            "2019-12-19 22:04:36.377132: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2019-12-19 22:04:36.377659: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2019-12-19 22:04:36.378160: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:39] Overriding allow_growth setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\n",
            "2019-12-19 22:04:36.378198: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1304] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 15216 MB memory) -> physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0)\n",
            "WARNING:tensorflow:From run.py:80: The name tf.global_variables_initializer is deprecated. Please use tf.compat.v1.global_variables_initializer instead.\n",
            "\n",
            "INFO:tensorflow:Training model for 10 epochs\n",
            "Epochs:   0% 0/10 [00:00<?, ?it/s]\n",
            "Batches:   0% 0/638 [00:00<?, ?it/s, acc=0]2019-12-19 22:04:37.889196: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0\n",
            "\u001b[A\n",
            "Batches:   0% 1/638 [00:00<09:30,  1.12it/s, dev_acc=0.49, epoch=0, loss=0.45, train_acc=0.50]\u001b[A\n",
            "Batches:   0% 2/638 [00:01<07:01,  1.51it/s, dev_acc=0.49, epoch=0, loss=0.45, train_acc=0.50]\u001b[A\n",
            "Batches:   0% 3/638 [00:01<05:19,  1.99it/s, dev_acc=0.49, epoch=0, loss=0.45, train_acc=0.50]\u001b[A\n",
            "Batches:   1% 4/638 [00:01<04:08,  2.55it/s, dev_acc=0.49, epoch=0, loss=0.45, train_acc=0.50]\u001b[A\n",
            "Batches:   1% 5/638 [00:01<03:18,  3.20it/s, dev_acc=0.49, epoch=0, loss=0.45, train_acc=0.50]\u001b[A\n",
            "Batches:   1% 6/638 [00:01<02:44,  3.85it/s, dev_acc=0.49, epoch=0, loss=0.45, train_acc=0.50]\u001b[A\n",
            "Batches:   1% 7/638 [00:01<02:20,  4.50it/s, dev_acc=0.49, epoch=0, loss=0.45, train_acc=0.50]\u001b[A\n",
            "Batches:   1% 8/638 [00:01<02:04,  5.07it/s, dev_acc=0.49, epoch=0, loss=0.45, train_acc=0.50]\u001b[A\n",
            "Batches:   1% 9/638 [00:01<01:51,  5.65it/s, dev_acc=0.49, epoch=0, loss=0.45, train_acc=0.50]\u001b[A\n",
            "Batches:   2% 10/638 [00:02<01:42,  6.14it/s, dev_acc=0.49, epoch=0, loss=0.45, train_acc=0.50]\u001b[A\n",
            "Batches:   2% 11/638 [00:02<01:34,  6.63it/s, dev_acc=0.49, epoch=0, loss=0.45, train_acc=0.50]\u001b[A\n",
            "Batches:   2% 12/638 [00:02<01:27,  7.14it/s, dev_acc=0.49, epoch=0, loss=0.45, train_acc=0.50]\u001b[A\n",
            "Batches:   2% 13/638 [00:02<01:27,  7.11it/s, dev_acc=0.49, epoch=0, loss=0.45, train_acc=0.50]\u001b[A\n",
            "Batches:   2% 14/638 [00:02<01:28,  7.04it/s, dev_acc=0.49, epoch=0, loss=0.45, train_acc=0.50]\u001b[A\n",
            "Batches:   2% 15/638 [00:02<01:28,  7.05it/s, dev_acc=0.49, epoch=0, loss=0.45, train_acc=0.50]\u001b[A\n",
            "Batches:   3% 16/638 [00:02<01:27,  7.12it/s, dev_acc=0.49, epoch=0, loss=0.45, train_acc=0.50]\u001b[A\n",
            "Batches:   3% 17/638 [00:02<01:22,  7.49it/s, dev_acc=0.49, epoch=0, loss=0.45, train_acc=0.50]\u001b[A\n",
            "Batches:   3% 18/638 [00:03<01:23,  7.44it/s, dev_acc=0.49, epoch=0, loss=0.45, train_acc=0.50]\u001b[A\n",
            "Batches:   3% 19/638 [00:03<01:22,  7.48it/s, dev_acc=0.49, epoch=0, loss=0.45, train_acc=0.50]\u001b[A\n",
            "Batches:   3% 20/638 [00:03<01:22,  7.48it/s, dev_acc=0.49, epoch=0, loss=0.45, train_acc=0.50]\u001b[A\n",
            "Batches:   3% 21/638 [00:03<01:48,  5.68it/s, dev_acc=0.57, epoch=0, loss=0.28, train_acc=0.57]\u001b[A\n",
            "Batches:   3% 22/638 [00:03<01:36,  6.36it/s, dev_acc=0.57, epoch=0, loss=0.28, train_acc=0.57]\u001b[A\n",
            "Batches:   4% 23/638 [00:03<01:32,  6.65it/s, dev_acc=0.57, epoch=0, loss=0.28, train_acc=0.57]\u001b[A\n",
            "Batches:   4% 24/638 [00:04<01:29,  6.86it/s, dev_acc=0.57, epoch=0, loss=0.28, train_acc=0.57]\u001b[A\n",
            "Batches:   4% 25/638 [00:04<01:27,  6.99it/s, dev_acc=0.57, epoch=0, loss=0.28, train_acc=0.57]\u001b[A\n",
            "Batches:   4% 26/638 [00:04<01:25,  7.15it/s, dev_acc=0.57, epoch=0, loss=0.28, train_acc=0.57]\u001b[A\n",
            "Batches:   4% 27/638 [00:04<01:23,  7.29it/s, dev_acc=0.57, epoch=0, loss=0.28, train_acc=0.57]\u001b[A\n",
            "Batches:   4% 28/638 [00:04<01:22,  7.36it/s, dev_acc=0.57, epoch=0, loss=0.28, train_acc=0.57]\u001b[A\n",
            "Batches:   5% 29/638 [00:04<01:24,  7.25it/s, dev_acc=0.57, epoch=0, loss=0.28, train_acc=0.57]\u001b[A\n",
            "Batches:   5% 30/638 [00:04<01:22,  7.37it/s, dev_acc=0.57, epoch=0, loss=0.28, train_acc=0.57]\u001b[A\n",
            "Batches:   5% 31/638 [00:04<01:22,  7.38it/s, dev_acc=0.57, epoch=0, loss=0.28, train_acc=0.57]\u001b[A\n",
            "Batches:   5% 32/638 [00:05<01:18,  7.70it/s, dev_acc=0.57, epoch=0, loss=0.28, train_acc=0.57]\u001b[A\n",
            "Batches:   5% 33/638 [00:05<01:18,  7.68it/s, dev_acc=0.57, epoch=0, loss=0.28, train_acc=0.57]\u001b[A\n",
            "Batches:   5% 34/638 [00:05<01:19,  7.61it/s, dev_acc=0.57, epoch=0, loss=0.28, train_acc=0.57]\u001b[A\n",
            "Batches:   5% 35/638 [00:05<01:21,  7.38it/s, dev_acc=0.57, epoch=0, loss=0.28, train_acc=0.57]\u001b[A\n",
            "Batches:   6% 36/638 [00:05<01:22,  7.34it/s, dev_acc=0.57, epoch=0, loss=0.28, train_acc=0.57]\u001b[A\n",
            "Batches:   6% 37/638 [00:05<01:20,  7.43it/s, dev_acc=0.57, epoch=0, loss=0.28, train_acc=0.57]\u001b[A\n",
            "Batches:   6% 38/638 [00:05<01:19,  7.59it/s, dev_acc=0.57, epoch=0, loss=0.28, train_acc=0.57]\u001b[A\n",
            "Batches:   6% 39/638 [00:06<01:18,  7.60it/s, dev_acc=0.57, epoch=0, loss=0.28, train_acc=0.57]\u001b[A\n",
            "Batches:   6% 40/638 [00:06<01:19,  7.52it/s, dev_acc=0.57, epoch=0, loss=0.28, train_acc=0.57]\u001b[A\n",
            "Batches:   6% 41/638 [00:06<01:47,  5.57it/s, dev_acc=0.62, epoch=0, loss=0.25, train_acc=0.63]\u001b[A\n",
            "Batches:   7% 42/638 [00:06<01:35,  6.23it/s, dev_acc=0.62, epoch=0, loss=0.25, train_acc=0.63]\u001b[A\n",
            "Batches:   7% 43/638 [00:06<01:31,  6.49it/s, dev_acc=0.62, epoch=0, loss=0.25, train_acc=0.63]\u001b[A\n",
            "Batches:   7% 44/638 [00:06<01:29,  6.63it/s, dev_acc=0.62, epoch=0, loss=0.25, train_acc=0.63]\u001b[A\n",
            "Batches:   7% 45/638 [00:07<01:26,  6.89it/s, dev_acc=0.62, epoch=0, loss=0.25, train_acc=0.63]\u001b[A\n",
            "Batches:   7% 46/638 [00:07<01:23,  7.13it/s, dev_acc=0.62, epoch=0, loss=0.25, train_acc=0.63]\u001b[A\n",
            "Batches:   7% 47/638 [00:07<01:21,  7.22it/s, dev_acc=0.62, epoch=0, loss=0.25, train_acc=0.63]\u001b[A\n",
            "Batches:   8% 48/638 [00:07<01:17,  7.60it/s, dev_acc=0.62, epoch=0, loss=0.25, train_acc=0.63]\u001b[A\n",
            "Batches:   8% 49/638 [00:07<01:16,  7.66it/s, dev_acc=0.62, epoch=0, loss=0.25, train_acc=0.63]\u001b[A\n",
            "Batches:   8% 50/638 [00:07<01:16,  7.64it/s, dev_acc=0.62, epoch=0, loss=0.25, train_acc=0.63]\u001b[A\n",
            "Batches:   8% 51/638 [00:07<01:17,  7.58it/s, dev_acc=0.62, epoch=0, loss=0.25, train_acc=0.63]\u001b[A\n",
            "Batches:   8% 52/638 [00:07<01:19,  7.40it/s, dev_acc=0.62, epoch=0, loss=0.25, train_acc=0.63]\u001b[A\n",
            "Batches:   8% 53/638 [00:08<01:20,  7.27it/s, dev_acc=0.62, epoch=0, loss=0.25, train_acc=0.63]\u001b[A\n",
            "Batches:   8% 54/638 [00:08<01:20,  7.26it/s, dev_acc=0.62, epoch=0, loss=0.25, train_acc=0.63]\u001b[A\n",
            "Batches:   9% 55/638 [00:08<01:20,  7.23it/s, dev_acc=0.62, epoch=0, loss=0.25, train_acc=0.63]\u001b[A\n",
            "Batches:   9% 56/638 [00:08<01:19,  7.28it/s, dev_acc=0.62, epoch=0, loss=0.25, train_acc=0.63]\u001b[A\n",
            "Batches:   9% 57/638 [00:08<01:21,  7.13it/s, dev_acc=0.62, epoch=0, loss=0.25, train_acc=0.63]\u001b[A\n",
            "Batches:   9% 58/638 [00:08<01:21,  7.12it/s, dev_acc=0.62, epoch=0, loss=0.25, train_acc=0.63]\u001b[A\n",
            "Batches:   9% 59/638 [00:08<01:21,  7.14it/s, dev_acc=0.62, epoch=0, loss=0.25, train_acc=0.63]\u001b[A\n",
            "Batches:   9% 60/638 [00:09<01:22,  6.99it/s, dev_acc=0.62, epoch=0, loss=0.25, train_acc=0.63]\u001b[A\n",
            "Batches:  10% 61/638 [00:09<01:45,  5.48it/s, dev_acc=0.64, epoch=0, loss=0.23, train_acc=0.64]\u001b[A\n",
            "Batches:  10% 62/638 [00:09<01:37,  5.90it/s, dev_acc=0.64, epoch=0, loss=0.23, train_acc=0.64]\u001b[A\n",
            "Batches:  10% 63/638 [00:09<01:35,  6.04it/s, dev_acc=0.64, epoch=0, loss=0.23, train_acc=0.64]\u001b[A\n",
            "Batches:  10% 64/638 [00:09<01:31,  6.28it/s, dev_acc=0.64, epoch=0, loss=0.23, train_acc=0.64]\u001b[A\n",
            "Batches:  10% 65/638 [00:09<01:24,  6.81it/s, dev_acc=0.64, epoch=0, loss=0.23, train_acc=0.64]\u001b[A\n",
            "Batches:  10% 66/638 [00:10<01:22,  6.91it/s, dev_acc=0.64, epoch=0, loss=0.23, train_acc=0.64]\u001b[A\n",
            "Batches:  11% 67/638 [00:10<01:20,  7.05it/s, dev_acc=0.64, epoch=0, loss=0.23, train_acc=0.64]\u001b[A\n",
            "Batches:  11% 68/638 [00:10<01:18,  7.22it/s, dev_acc=0.64, epoch=0, loss=0.23, train_acc=0.64]\u001b[A\n",
            "Batches:  11% 69/638 [00:10<01:19,  7.20it/s, dev_acc=0.64, epoch=0, loss=0.23, train_acc=0.64]\u001b[A\n",
            "Batches:  11% 70/638 [00:10<01:20,  7.09it/s, dev_acc=0.64, epoch=0, loss=0.23, train_acc=0.64]\u001b[A\n",
            "Batches:  11% 71/638 [00:10<01:19,  7.17it/s, dev_acc=0.64, epoch=0, loss=0.23, train_acc=0.64]\u001b[A\n",
            "Batches:  11% 72/638 [00:10<01:17,  7.26it/s, dev_acc=0.64, epoch=0, loss=0.23, train_acc=0.64]\u001b[A\n",
            "Batches:  11% 73/638 [00:10<01:17,  7.31it/s, dev_acc=0.64, epoch=0, loss=0.23, train_acc=0.64]\u001b[A\n",
            "Batches:  12% 74/638 [00:11<01:17,  7.31it/s, dev_acc=0.64, epoch=0, loss=0.23, train_acc=0.64]\u001b[A\n",
            "Batches:  12% 75/638 [00:11<01:16,  7.37it/s, dev_acc=0.64, epoch=0, loss=0.23, train_acc=0.64]\u001b[A\n",
            "Batches:  12% 76/638 [00:11<01:16,  7.37it/s, dev_acc=0.64, epoch=0, loss=0.23, train_acc=0.64]\u001b[A\n",
            "Batches:  12% 77/638 [00:11<01:16,  7.38it/s, dev_acc=0.64, epoch=0, loss=0.23, train_acc=0.64]\u001b[A\n",
            "Batches:  12% 78/638 [00:11<01:20,  6.99it/s, dev_acc=0.64, epoch=0, loss=0.23, train_acc=0.64]\u001b[A\n",
            "Batches:  12% 79/638 [00:11<01:18,  7.15it/s, dev_acc=0.64, epoch=0, loss=0.23, train_acc=0.64]\u001b[A\n",
            "Batches:  13% 80/638 [00:11<01:14,  7.53it/s, dev_acc=0.64, epoch=0, loss=0.23, train_acc=0.64]\u001b[A\n",
            "Batches:  13% 81/638 [00:12<01:39,  5.61it/s, dev_acc=0.66, epoch=0, loss=0.21, train_acc=0.67]\u001b[A\n",
            "Batches:  13% 82/638 [00:12<01:32,  6.01it/s, dev_acc=0.66, epoch=0, loss=0.21, train_acc=0.67]\u001b[A\n",
            "Batches:  13% 83/638 [00:12<01:27,  6.36it/s, dev_acc=0.66, epoch=0, loss=0.21, train_acc=0.67]\u001b[A\n",
            "Batches:  13% 84/638 [00:12<01:21,  6.81it/s, dev_acc=0.66, epoch=0, loss=0.21, train_acc=0.67]\u001b[A\n",
            "Batches:  13% 85/638 [00:12<01:20,  6.88it/s, dev_acc=0.66, epoch=0, loss=0.21, train_acc=0.67]\u001b[A\n",
            "Batches:  13% 86/638 [00:12<01:20,  6.83it/s, dev_acc=0.66, epoch=0, loss=0.21, train_acc=0.67]\u001b[A\n",
            "Batches:  14% 87/638 [00:13<01:21,  6.79it/s, dev_acc=0.66, epoch=0, loss=0.21, train_acc=0.67]\u001b[A\n",
            "Batches:  14% 88/638 [00:13<01:20,  6.81it/s, dev_acc=0.66, epoch=0, loss=0.21, train_acc=0.67]\u001b[A\n",
            "Batches:  14% 89/638 [00:13<01:17,  7.12it/s, dev_acc=0.66, epoch=0, loss=0.21, train_acc=0.67]\u001b[A\n",
            "Batches:  14% 90/638 [00:13<01:13,  7.45it/s, dev_acc=0.66, epoch=0, loss=0.21, train_acc=0.67]\u001b[A\n",
            "Batches:  14% 91/638 [00:13<01:15,  7.25it/s, dev_acc=0.66, epoch=0, loss=0.21, train_acc=0.67]\u001b[A\n",
            "Batches:  14% 92/638 [00:13<01:14,  7.29it/s, dev_acc=0.66, epoch=0, loss=0.21, train_acc=0.67]\u001b[A\n",
            "Batches:  15% 93/638 [00:13<01:16,  7.11it/s, dev_acc=0.66, epoch=0, loss=0.21, train_acc=0.67]\u001b[A\n",
            "Batches:  15% 94/638 [00:14<01:16,  7.14it/s, dev_acc=0.66, epoch=0, loss=0.21, train_acc=0.67]\u001b[A\n",
            "Batches:  15% 95/638 [00:14<01:14,  7.28it/s, dev_acc=0.66, epoch=0, loss=0.21, train_acc=0.67]\u001b[A\n",
            "Batches:  15% 96/638 [00:14<01:15,  7.23it/s, dev_acc=0.66, epoch=0, loss=0.21, train_acc=0.67]\u001b[A\n",
            "Batches:  15% 97/638 [00:14<01:12,  7.51it/s, dev_acc=0.66, epoch=0, loss=0.21, train_acc=0.67]\u001b[A\n",
            "Batches:  15% 98/638 [00:14<01:12,  7.41it/s, dev_acc=0.66, epoch=0, loss=0.21, train_acc=0.67]\u001b[A\n",
            "Batches:  16% 99/638 [00:14<01:12,  7.45it/s, dev_acc=0.66, epoch=0, loss=0.21, train_acc=0.67]\u001b[A\n",
            "Batches:  16% 100/638 [00:14<01:35,  5.63it/s, dev_acc=0.66, epoch=0, loss=0.21, train_acc=0.67]\u001b[A\n",
            "Batches:  16% 101/638 [00:15<01:50,  4.88it/s, dev_acc=0.68, epoch=0, loss=0.21, train_acc=0.69]\u001b[A\n",
            "Batches:  16% 102/638 [00:15<01:39,  5.39it/s, dev_acc=0.68, epoch=0, loss=0.21, train_acc=0.69]\u001b[A\n",
            "Batches:  16% 103/638 [00:15<01:30,  5.89it/s, dev_acc=0.68, epoch=0, loss=0.21, train_acc=0.69]\u001b[A\n",
            "Batches:  16% 104/638 [00:15<01:25,  6.22it/s, dev_acc=0.68, epoch=0, loss=0.21, train_acc=0.69]\u001b[A\n",
            "Batches:  16% 105/638 [00:15<01:22,  6.46it/s, dev_acc=0.68, epoch=0, loss=0.21, train_acc=0.69]\u001b[A\n",
            "Batches:  17% 106/638 [00:15<01:17,  6.85it/s, dev_acc=0.68, epoch=0, loss=0.21, train_acc=0.69]\u001b[A\n",
            "Batches:  17% 107/638 [00:16<01:15,  7.03it/s, dev_acc=0.68, epoch=0, loss=0.21, train_acc=0.69]\u001b[A\n",
            "Batches:  17% 108/638 [00:16<01:13,  7.24it/s, dev_acc=0.68, epoch=0, loss=0.21, train_acc=0.69]\u001b[A\n",
            "Batches:  17% 109/638 [00:16<01:10,  7.52it/s, dev_acc=0.68, epoch=0, loss=0.21, train_acc=0.69]\u001b[A\n",
            "Batches:  17% 110/638 [00:16<01:11,  7.41it/s, dev_acc=0.68, epoch=0, loss=0.21, train_acc=0.69]\u001b[A\n",
            "Batches:  17% 111/638 [00:16<01:10,  7.53it/s, dev_acc=0.68, epoch=0, loss=0.21, train_acc=0.69]\u001b[A\n",
            "Batches:  18% 112/638 [00:16<01:09,  7.59it/s, dev_acc=0.68, epoch=0, loss=0.21, train_acc=0.69]\u001b[A\n",
            "Batches:  18% 113/638 [00:16<01:11,  7.36it/s, dev_acc=0.68, epoch=0, loss=0.21, train_acc=0.69]\u001b[A\n",
            "Batches:  18% 114/638 [00:16<01:09,  7.50it/s, dev_acc=0.68, epoch=0, loss=0.21, train_acc=0.69]\u001b[A\n",
            "Batches:  18% 115/638 [00:17<01:10,  7.46it/s, dev_acc=0.68, epoch=0, loss=0.21, train_acc=0.69]\u001b[A\n",
            "Batches:  18% 116/638 [00:17<01:10,  7.40it/s, dev_acc=0.68, epoch=0, loss=0.21, train_acc=0.69]\u001b[A\n",
            "Batches:  18% 117/638 [00:17<01:09,  7.45it/s, dev_acc=0.68, epoch=0, loss=0.21, train_acc=0.69]\u001b[A\n",
            "Batches:  18% 118/638 [00:17<01:11,  7.26it/s, dev_acc=0.68, epoch=0, loss=0.21, train_acc=0.69]\u001b[A\n",
            "Batches:  19% 119/638 [00:17<01:12,  7.13it/s, dev_acc=0.68, epoch=0, loss=0.21, train_acc=0.69]\u001b[A\n",
            "Batches:  19% 120/638 [00:17<01:13,  7.08it/s, dev_acc=0.68, epoch=0, loss=0.21, train_acc=0.69]\u001b[A\n",
            "Batches:  19% 121/638 [00:18<01:35,  5.42it/s, dev_acc=0.69, epoch=0, loss=0.20, train_acc=0.70]\u001b[A\n",
            "Batches:  19% 122/638 [00:18<01:24,  6.10it/s, dev_acc=0.69, epoch=0, loss=0.20, train_acc=0.70]\u001b[A\n",
            "Batches:  19% 123/638 [00:18<01:19,  6.44it/s, dev_acc=0.69, epoch=0, loss=0.20, train_acc=0.70]\u001b[A\n",
            "Batches:  19% 124/638 [00:18<01:17,  6.59it/s, dev_acc=0.69, epoch=0, loss=0.20, train_acc=0.70]\u001b[A\n",
            "Batches:  20% 125/638 [00:18<01:13,  7.00it/s, dev_acc=0.69, epoch=0, loss=0.20, train_acc=0.70]\u001b[A\n",
            "Batches:  20% 126/638 [00:18<01:13,  6.96it/s, dev_acc=0.69, epoch=0, loss=0.20, train_acc=0.70]\u001b[A\n",
            "Batches:  20% 127/638 [00:18<01:12,  7.06it/s, dev_acc=0.69, epoch=0, loss=0.20, train_acc=0.70]\u001b[A\n",
            "Batches:  20% 128/638 [00:19<01:10,  7.22it/s, dev_acc=0.69, epoch=0, loss=0.20, train_acc=0.70]\u001b[A\n",
            "Batches:  20% 129/638 [00:19<01:10,  7.20it/s, dev_acc=0.69, epoch=0, loss=0.20, train_acc=0.70]\u001b[A\n",
            "Batches:  20% 130/638 [00:19<01:09,  7.35it/s, dev_acc=0.69, epoch=0, loss=0.20, train_acc=0.70]\u001b[A\n",
            "Batches:  21% 131/638 [00:19<01:06,  7.66it/s, dev_acc=0.69, epoch=0, loss=0.20, train_acc=0.70]\u001b[A\n",
            "Batches:  21% 132/638 [00:19<01:08,  7.39it/s, dev_acc=0.69, epoch=0, loss=0.20, train_acc=0.70]\u001b[A\n",
            "Batches:  21% 133/638 [00:19<01:04,  7.78it/s, dev_acc=0.69, epoch=0, loss=0.20, train_acc=0.70]\u001b[A\n",
            "Batches:  21% 134/638 [00:19<01:06,  7.56it/s, dev_acc=0.69, epoch=0, loss=0.20, train_acc=0.70]\u001b[A\n",
            "Batches:  21% 135/638 [00:19<01:08,  7.29it/s, dev_acc=0.69, epoch=0, loss=0.20, train_acc=0.70]\u001b[A\n",
            "Batches:  21% 136/638 [00:20<01:05,  7.64it/s, dev_acc=0.69, epoch=0, loss=0.20, train_acc=0.70]\u001b[A\n",
            "Batches:  21% 137/638 [00:20<01:03,  7.94it/s, dev_acc=0.69, epoch=0, loss=0.20, train_acc=0.70]\u001b[A\n",
            "Batches:  22% 138/638 [00:20<01:02,  8.06it/s, dev_acc=0.69, epoch=0, loss=0.20, train_acc=0.70]\u001b[A\n",
            "Batches:  22% 139/638 [00:20<01:03,  7.84it/s, dev_acc=0.69, epoch=0, loss=0.20, train_acc=0.70]\u001b[A\n",
            "Batches:  22% 140/638 [00:20<01:02,  7.97it/s, dev_acc=0.69, epoch=0, loss=0.20, train_acc=0.70]\u001b[A\n",
            "Batches:  22% 141/638 [00:20<01:26,  5.72it/s, dev_acc=0.70, epoch=0, loss=0.19, train_acc=0.72]\u001b[A\n",
            "Batches:  22% 142/638 [00:20<01:21,  6.07it/s, dev_acc=0.70, epoch=0, loss=0.19, train_acc=0.72]\u001b[A\n",
            "Batches:  22% 143/638 [00:21<01:16,  6.51it/s, dev_acc=0.70, epoch=0, loss=0.19, train_acc=0.72]\u001b[A\n",
            "Batches:  23% 144/638 [00:21<01:13,  6.76it/s, dev_acc=0.70, epoch=0, loss=0.19, train_acc=0.72]\u001b[A\n",
            "Batches:  23% 145/638 [00:21<01:10,  6.95it/s, dev_acc=0.70, epoch=0, loss=0.19, train_acc=0.72]\u001b[A\n",
            "Batches:  23% 146/638 [00:21<01:07,  7.25it/s, dev_acc=0.70, epoch=0, loss=0.19, train_acc=0.72]\u001b[A\n",
            "Batches:  23% 147/638 [00:21<01:07,  7.22it/s, dev_acc=0.70, epoch=0, loss=0.19, train_acc=0.72]\u001b[A\n",
            "Batches:  23% 148/638 [00:21<01:07,  7.21it/s, dev_acc=0.70, epoch=0, loss=0.19, train_acc=0.72]\u001b[A\n",
            "Batches:  23% 149/638 [00:21<01:08,  7.18it/s, dev_acc=0.70, epoch=0, loss=0.19, train_acc=0.72]\u001b[A\n",
            "Batches:  24% 150/638 [00:22<01:07,  7.20it/s, dev_acc=0.70, epoch=0, loss=0.19, train_acc=0.72]\u001b[A\n",
            "Batches:  24% 151/638 [00:22<01:03,  7.66it/s, dev_acc=0.70, epoch=0, loss=0.19, train_acc=0.72]\u001b[A\n",
            "Batches:  24% 152/638 [00:22<01:03,  7.61it/s, dev_acc=0.70, epoch=0, loss=0.19, train_acc=0.72]\u001b[A\n",
            "Batches:  24% 153/638 [00:22<01:06,  7.29it/s, dev_acc=0.70, epoch=0, loss=0.19, train_acc=0.72]\u001b[A\n",
            "Batches:  24% 154/638 [00:22<01:05,  7.38it/s, dev_acc=0.70, epoch=0, loss=0.19, train_acc=0.72]\u001b[A\n",
            "Batches:  24% 155/638 [00:22<01:06,  7.29it/s, dev_acc=0.70, epoch=0, loss=0.19, train_acc=0.72]\u001b[A\n",
            "Batches:  24% 156/638 [00:22<01:08,  7.05it/s, dev_acc=0.70, epoch=0, loss=0.19, train_acc=0.72]\u001b[A\n",
            "Batches:  25% 157/638 [00:23<01:06,  7.20it/s, dev_acc=0.70, epoch=0, loss=0.19, train_acc=0.72]\u001b[A\n",
            "Batches:  25% 158/638 [00:23<01:06,  7.27it/s, dev_acc=0.70, epoch=0, loss=0.19, train_acc=0.72]\u001b[A\n",
            "Batches:  25% 159/638 [00:23<01:02,  7.66it/s, dev_acc=0.70, epoch=0, loss=0.19, train_acc=0.72]\u001b[A\n",
            "Batches:  25% 160/638 [00:23<01:04,  7.44it/s, dev_acc=0.70, epoch=0, loss=0.19, train_acc=0.72]\u001b[A\n",
            "Batches:  25% 161/638 [00:23<01:24,  5.65it/s, dev_acc=0.71, epoch=0, loss=0.20, train_acc=0.73]\u001b[A\n",
            "Batches:  25% 162/638 [00:23<01:18,  6.04it/s, dev_acc=0.71, epoch=0, loss=0.20, train_acc=0.73]\u001b[A\n",
            "Batches:  26% 163/638 [00:23<01:14,  6.34it/s, dev_acc=0.71, epoch=0, loss=0.20, train_acc=0.73]\u001b[A\n",
            "Batches:  26% 164/638 [00:24<01:10,  6.69it/s, dev_acc=0.71, epoch=0, loss=0.20, train_acc=0.73]\u001b[A\n",
            "Batches:  26% 165/638 [00:24<01:09,  6.82it/s, dev_acc=0.71, epoch=0, loss=0.20, train_acc=0.73]\u001b[A\n",
            "Batches:  26% 166/638 [00:24<01:07,  6.96it/s, dev_acc=0.71, epoch=0, loss=0.20, train_acc=0.73]\u001b[A\n",
            "Batches:  26% 167/638 [00:24<01:07,  6.98it/s, dev_acc=0.71, epoch=0, loss=0.20, train_acc=0.73]\u001b[A\n",
            "Batches:  26% 168/638 [00:24<01:07,  6.98it/s, dev_acc=0.71, epoch=0, loss=0.20, train_acc=0.73]\u001b[A\n",
            "Batches:  26% 169/638 [00:24<01:04,  7.22it/s, dev_acc=0.71, epoch=0, loss=0.20, train_acc=0.73]\u001b[A\n",
            "Batches:  27% 170/638 [00:24<01:06,  7.08it/s, dev_acc=0.71, epoch=0, loss=0.20, train_acc=0.73]\u001b[A\n",
            "Batches:  27% 171/638 [00:25<01:02,  7.45it/s, dev_acc=0.71, epoch=0, loss=0.20, train_acc=0.73]\u001b[A\n",
            "Batches:  27% 172/638 [00:25<01:03,  7.36it/s, dev_acc=0.71, epoch=0, loss=0.20, train_acc=0.73]\u001b[A\n",
            "Batches:  27% 173/638 [00:25<01:03,  7.34it/s, dev_acc=0.71, epoch=0, loss=0.20, train_acc=0.73]"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yqbOVJG4HGbV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!cd multihead-siamese-nets && python3 run.py train multihead SNLI"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}